{
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.12",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "sourceId": 1445566,
     "sourceType": "datasetVersion",
     "datasetId": 847319
    }
   ],
   "dockerImageVersionId": 30627,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook",
   "isGpuEnabled": true
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "np.random.seed(seed)\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:09:16.444275Z",
     "iopub.execute_input": "2023-12-15T07:09:16.444669Z",
     "iopub.status.idle": "2023-12-15T07:09:18.953217Z",
     "shell.execute_reply.started": "2023-12-15T07:09:16.444636Z",
     "shell.execute_reply": "2023-12-15T07:09:18.952225Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": "sys.version_info(major=3, minor=10, micro=12, releaselevel='final', serial=0)\nmatplotlib 3.7.4\nnumpy 1.24.3\npandas 2.1.4\nsklearn 1.2.2\ntorch 2.0.0\ncuda:0\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 数据加载"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "import unicodedata\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#因为西班牙语有一些是特殊字符，所以我们需要unicode转ascii，\n",
    "# 这样值变小了，因为unicode太大\n",
    "def unicode_to_ascii(s):\n",
    "    #NFD是转换方法，把每一个字节拆开，Mn是重音，所以去除\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "#下面我们找个样本测试一下\n",
    "# 加u代表对字符串进行unicode编码\n",
    "en_sentence = u\"May I borrow this book?\"\n",
    "sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n",
    "\n",
    "print(unicode_to_ascii(en_sentence))\n",
    "print(unicode_to_ascii(sp_sentence))\n",
    "\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "    #变为小写，去掉多余的空格\n",
    "    w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "    # 在单词与跟在其后的标点符号之间插入一个空格\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    #因为可能有多余空格，替换为一个空格，所以处理一下\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "    # 除了 (a-z, A-Z, \".\", \"?\", \"!\", \",\")，将所有字符替换为空格\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "\n",
    "    w = w.rstrip().strip()\n",
    "\n",
    "    return w\n",
    "\n",
    "print(preprocess_sentence(en_sentence))\n",
    "print(preprocess_sentence(sp_sentence))\n",
    "print(preprocess_sentence(sp_sentence).encode('utf-8'))  #¿是占用两个字节的"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:09:18.954769Z",
     "iopub.execute_input": "2023-12-15T07:09:18.955190Z",
     "iopub.status.idle": "2023-12-15T07:09:19.073003Z",
     "shell.execute_reply.started": "2023-12-15T07:09:18.955162Z",
     "shell.execute_reply": "2023-12-15T07:09:19.072053Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "text": "May I borrow this book?\n¿Puedo tomar prestado este libro?\nmay i borrow this book ?\n¿ puedo tomar prestado este libro ?\nb'\\xc2\\xbf puedo tomar prestado este libro ?'\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Dataset"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "!wc ../input/data-spa/spa.txt -l"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:09:19.074164Z",
     "iopub.execute_input": "2023-12-15T07:09:19.074483Z",
     "iopub.status.idle": "2023-12-15T07:09:20.183498Z",
     "shell.execute_reply.started": "2023-12-15T07:09:19.074458Z",
     "shell.execute_reply": "2023-12-15T07:09:20.182111Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "text": "118964 ../input/data-spa/spa.txt\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "split_index = np.random.choice(a=[\"train\", \"test\"], replace=True, p=[0.9, 0.1], size=118964)\n",
    "split_index=='train'"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:09:20.186038Z",
     "iopub.execute_input": "2023-12-15T07:09:20.186388Z",
     "iopub.status.idle": "2023-12-15T07:09:20.206713Z",
     "shell.execute_reply.started": "2023-12-15T07:09:20.186343Z",
     "shell.execute_reply": "2023-12-15T07:09:20.205794Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "execution_count": 5,
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([ True, False,  True, ...,  True,  True,  True])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class LangPairDataset(Dataset):\n",
    "    fpath = Path(r\"../input/data-spa/spa.txt\")\n",
    "    cache_path = Path(r\"./.cache/lang_pair.npy\")\n",
    "    split_index = np.random.choice(a=[\"train\", \"test\"], replace=True, p=[0.9, 0.1], size=118964) #按照9:1划分训练集和测试集\n",
    "    def __init__(self, mode=\"train\", cache=False):\n",
    "        if cache or not self.cache_path.exists():\n",
    "            self.cache_path.parent.mkdir(parents=True, exist_ok=True)\n",
    "            with open(self.fpath, \"r\", encoding=\"utf8\") as file:\n",
    "                lines = file.readlines()\n",
    "                lang_pair = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines]\n",
    "                trg, src = zip(*lang_pair)\n",
    "                print(len(trg), trg[0])\n",
    "                print(src[0])\n",
    "                np.save(self.cache_path, {\"trg\": np.array(trg), \"src\": np.array(src)}) #保存为npy文件,方便下次直接读取,不用再处理\n",
    "        else:\n",
    "            lang_pair = np.load(self.cache_path, allow_pickle=True).item() #读取npy文件，allow_pickle=True允许读取字典\n",
    "            trg = lang_pair[\"trg\"]\n",
    "            src = lang_pair[\"src\"]\n",
    "        \n",
    "        self.trg = trg[self.split_index == mode] #按照index拿到训练集的标签\n",
    "        self.src = src[self.split_index == mode] #按照index拿到训练集的源语言\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.src[index], self.trg[index]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "        \n",
    "\n",
    "train_ds = LangPairDataset(\"train\")\n",
    "test_ds = LangPairDataset(\"test\")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:05.675619Z",
     "iopub.execute_input": "2023-12-15T07:10:05.676030Z",
     "iopub.status.idle": "2023-12-15T07:10:06.196040Z",
     "shell.execute_reply.started": "2023-12-15T07:10:05.675996Z",
     "shell.execute_reply": "2023-12-15T07:10:06.195203Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "print(\"source: {}\\ntarget: {}\".format(*train_ds[-1]))"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:06.198055Z",
     "iopub.execute_input": "2023-12-15T07:10:06.198425Z",
     "iopub.status.idle": "2023-12-15T07:10:06.203981Z",
     "shell.execute_reply.started": "2023-12-15T07:10:06.198381Z",
     "shell.execute_reply": "2023-12-15T07:10:06.202993Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "text": "source: si quieres sonar como un hablante nativo , debes estar dispuesto a practicar diciendo la misma frase una y otra vez de la misma manera en que un musico de banjo practica el mismo fraseo una y otra vez hasta que lo puedan tocar correctamente y en el tiempo esperado .\ntarget: if you want to sound like a native speaker , you must be willing to practice saying the same sentence over and over in the same way that banjo players practice the same phrase over and over until they can play it correctly and at the desired tempo .\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Tokenizer\n",
    "\n",
    "这里有两种处理方式，分别对应着 encoder 和 decoder 的 word embedding 是否共享，这里实现不共享的方案。"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "from collections import Counter\n",
    "\n",
    "def get_word_idx(ds, mode=\"src\", threshold=2):\n",
    "    #载入词表，看下词表长度，词表就像英语字典\n",
    "    word2idx = {\n",
    "        \"[PAD]\": 0,     # 填充 token\n",
    "        \"[BOS]\": 1,     # begin of sentence\n",
    "        \"[UNK]\": 2,     # 未知 token\n",
    "        \"[EOS]\": 3,     # end of sentence\n",
    "    }\n",
    "    idx2word = {value: key for key, value in word2idx.items()}\n",
    "    index = len(idx2word)\n",
    "    threshold = 1  # 出现次数低于此的token舍弃\n",
    "\n",
    "    word_list = \" \".join([pair[0 if mode==\"src\" else 1] for pair in ds]).split()\n",
    "    counter = Counter(word_list) #统计词频\n",
    "\n",
    "    for token, count in counter.items():\n",
    "        if count >= threshold:#出现次数大于阈值的token加入词表\n",
    "            word2idx[token] = index\n",
    "            idx2word[index] = token\n",
    "            index += 1\n",
    "                \n",
    "    return word2idx, idx2word\n",
    "\n",
    "src_word2idx, src_idx2word = get_word_idx(train_ds, \"src\") #源语言词表\n",
    "trg_word2idx, trg_idx2word = get_word_idx(train_ds, \"trg\") #目标语言词表"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:06.205231Z",
     "iopub.execute_input": "2023-12-15T07:10:06.205558Z",
     "iopub.status.idle": "2023-12-15T07:10:07.145525Z",
     "shell.execute_reply.started": "2023-12-15T07:10:06.205514Z",
     "shell.execute_reply": "2023-12-15T07:10:07.144759Z"
    },
    "trusted": true
   },
   "execution_count": 9,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self, word2idx, idx2word, max_length=500, pad_idx=0, bos_idx=1, eos_idx=3, unk_idx=2):\n",
    "        self.word2idx = word2idx\n",
    "        self.idx2word = idx2word\n",
    "        self.max_length = max_length\n",
    "        self.pad_idx = pad_idx\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.unk_idx = unk_idx\n",
    "    \n",
    "    def encode(self, text_list, padding_first=False, add_bos=True, add_eos=True, return_mask=False):\n",
    "        \"\"\"如果padding_first == True，则padding加载前面，否则加载后面\n",
    "        return_mask: 是否返回mask(掩码），mask用于指示哪些是padding的，哪些是真实的token\n",
    "        \"\"\"\n",
    "        max_length = min(self.max_length, add_eos + add_bos + max([len(text) for text in text_list]))\n",
    "        indices_list = []\n",
    "        for text in text_list:\n",
    "            indices = [self.word2idx.get(word, self.unk_idx) for word in text[:max_length - add_bos - add_eos]] #如果词表中没有这个词，就用unk_idx代替，indices是一个list,里面是每个词的index,也就是一个样本的index\n",
    "            if add_bos:\n",
    "                indices = [self.bos_idx] + indices\n",
    "            if add_eos:\n",
    "                indices = indices + [self.eos_idx]\n",
    "            if padding_first:\n",
    "                indices = [self.pad_idx] * (max_length - len(indices)) + indices\n",
    "            else:\n",
    "                indices = indices + [self.pad_idx] * (max_length - len(indices))\n",
    "            indices_list.append(indices)\n",
    "        input_ids = torch.tensor(indices_list) #转换为tensor\n",
    "        masks = (input_ids == self.pad_idx).to(dtype=torch.int64) #mask是一个和input_ids一样大小的tensor，里面的值是0或者1，0代表token，1代表padding\n",
    "        return input_ids if not return_mask else (input_ids, masks)\n",
    "    \n",
    "    \n",
    "    def decode(self, indices_list, remove_bos=True, remove_eos=True, remove_pad=True, split=False):\n",
    "        text_list = []\n",
    "        for indices in indices_list:\n",
    "            text = []\n",
    "            for index in indices:\n",
    "                word = self.idx2word.get(index, \"[UNK]\")\n",
    "                if remove_bos and word == \"[BOS]\":\n",
    "                    continue\n",
    "                if remove_eos and word == \"[EOS]\":\n",
    "                    break\n",
    "                if remove_pad and word == \"[PAD]\":\n",
    "                    break\n",
    "                text.append(word)\n",
    "            text_list.append(\" \".join(text) if not split else text)\n",
    "        return text_list\n",
    "    \n",
    "\n",
    "src_tokenizer = Tokenizer(word2idx=src_word2idx, idx2word=src_idx2word)\n",
    "trg_tokenizer = Tokenizer(word2idx=trg_word2idx, idx2word=trg_idx2word)\n",
    "\n",
    "trg_tokenizer.encode([[\"hello\"], [\"hello\", \"world\"]], add_bos=True, add_eos=False)\n",
    "raw_text = [\"hello world\".split(), \"tokenize text datas with batch\".split(), \"this is a test\".split()]\n",
    "indices = trg_tokenizer.encode(raw_text, padding_first=False, add_bos=False, add_eos=True)\n",
    "decode_text = trg_tokenizer.decode(indices.tolist(), remove_bos=False, remove_eos=False, remove_pad=False)\n",
    "print(\"raw text\"+'-'*10)\n",
    "for raw in raw_text:\n",
    "    print(raw)\n",
    "print(\"indices\"+'-'*10)\n",
    "for index in indices:\n",
    "    print(index)\n",
    "print(\"decode text\"+'-'*10)\n",
    "for decode in decode_text:\n",
    "    print(decode)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:43.331039Z",
     "iopub.execute_input": "2023-12-15T07:10:43.331674Z",
     "iopub.status.idle": "2023-12-15T07:10:43.356829Z",
     "shell.execute_reply.started": "2023-12-15T07:10:43.331640Z",
     "shell.execute_reply": "2023-12-15T07:10:43.355621Z"
    },
    "trusted": true
   },
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "text": "raw text----------\n['hello', 'world']\n['tokenize', 'text', 'datas', 'with', 'batch']\n['this', 'is', 'a', 'test']\nindices----------\ntensor([ 301, 3224,    3,    0,    0,    0])\ntensor([   2, 3888,    2,  547,    2,    3])\ntensor([ 124,  238,  110, 1260,    3,    0])\ndecode text----------\nhello world [EOS] [PAD] [PAD] [PAD]\n[UNK] text [UNK] with [UNK] [EOS]\nthis is a test [EOS] [PAD]\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### DataLoader"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "def collate_fct(batch):\n",
    "    src_words = [pair[0].split() for pair in batch]\n",
    "    trg_words = [pair[1].split() for pair in batch]\n",
    "    \n",
    "    # [PAD] [BOS] src [EOS]\n",
    "    encoder_inputs, encoder_inputs_mask = src_tokenizer.encode(\n",
    "        src_words, padding_first=True, add_bos=True, add_eos=True, return_mask=True\n",
    "        )\n",
    "    \n",
    "    # [BOS] trg [PAD]\n",
    "    decoder_inputs = trg_tokenizer.encode(\n",
    "        trg_words, padding_first=False, add_bos=True, add_eos=False, return_mask=False,\n",
    "        )\n",
    "    \n",
    "    # trg [EOS] [PAD]\n",
    "    decoder_labels, decoder_labels_mask = trg_tokenizer.encode(\n",
    "        trg_words, padding_first=False, add_bos=False, add_eos=True, return_mask=True\n",
    "        )\n",
    "\n",
    "    return {\n",
    "        \"encoder_inputs\": encoder_inputs.to(device=device),\n",
    "        \"encoder_inputs_mask\": encoder_inputs_mask.to(device=device),\n",
    "        \"decoder_inputs\": decoder_inputs.to(device=device),\n",
    "        \"decoder_labels\": decoder_labels.to(device=device),\n",
    "        \"decoder_labels_mask\": decoder_labels_mask.to(device=device),\n",
    "    }\n",
    "    "
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:43.523379Z",
     "iopub.execute_input": "2023-12-15T07:10:43.524043Z",
     "iopub.status.idle": "2023-12-15T07:10:43.532427Z",
     "shell.execute_reply.started": "2023-12-15T07:10:43.524011Z",
     "shell.execute_reply": "2023-12-15T07:10:43.531301Z"
    },
    "trusted": true
   },
   "execution_count": 21,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "sample_dl = DataLoader(train_ds, batch_size=2, shuffle=True, collate_fn=collate_fct)\n",
    "\n",
    "for batch in sample_dl:\n",
    "    for key, value in batch.items():\n",
    "        print(key)\n",
    "        print(value)\n",
    "    break"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:43.534221Z",
     "iopub.execute_input": "2023-12-15T07:10:43.534593Z",
     "iopub.status.idle": "2023-12-15T07:10:48.518847Z",
     "shell.execute_reply.started": "2023-12-15T07:10:43.534540Z",
     "shell.execute_reply": "2023-12-15T07:10:48.517841Z"
    },
    "trusted": true
   },
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "text": "encoder_inputs\ntensor([[   1,   51,  319,  496,  131, 3085,    5,    3],\n        [   1,   96,   37,  162,   85, 2953,    5,    3]], device='cuda:0')\nencoder_inputs_mask\ntensor([[0, 0, 0, 0, 0, 0, 0, 0],\n        [0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0')\ndecoder_inputs\ntensor([[   1,   31,   83,  656,  722, 4022,    5],\n        [   1,   51,  893,  120,   33, 1478,    5]], device='cuda:0')\ndecoder_labels\ntensor([[  31,   83,  656,  722, 4022,    5,    3],\n        [  51,  893,  120,   33, 1478,    5,    3]], device='cuda:0')\ndecoder_labels_mask\ntensor([[0, 0, 0, 0, 0, 0, 0],\n        [0, 0, 0, 0, 0, 0, 0]], device='cuda:0')\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 定义模型"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        vocab_size, \n",
    "        embedding_dim=256, \n",
    "        hidden_dim=1024, \n",
    "        num_layers=1,\n",
    "        ):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        \n",
    "    def forward(self, encoder_inputs):\n",
    "        # encoder_inputs.shape = [batch size, sequence length]\n",
    "        bs, seq_len = encoder_inputs.shape\n",
    "        embeds = self.embedding(encoder_inputs)\n",
    "        # embeds.shape = [batch size, sequence length, embedding_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, sequence length, hidden_dim]\n",
    "        return seq_output, hidden"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.519881Z",
     "iopub.execute_input": "2023-12-15T07:10:48.520170Z",
     "iopub.status.idle": "2023-12-15T07:10:48.527082Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.520144Z",
     "shell.execute_reply": "2023-12-15T07:10:48.525997Z"
    },
    "trusted": true
   },
   "execution_count": 23,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class BahdanauAttention(nn.Module):\n",
    "    def __init__(self, hidden_dim=1024):\n",
    "        super().__init__()\n",
    "        self.Wk = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.Wq = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "        \n",
    "    def forward(self, query, keys, values, attn_mask=None):\n",
    "        # query.shape = [batch size, hidden_dim]\n",
    "        # keys.shape = [batch size, sequence length, hidden_dim]\n",
    "        # values.shape = [batch size, sequence length, hidden_dim]\n",
    "        # attention_mask.shape = [batch size, sequence length]\n",
    "        scores = self.V(F.tanh(self.Wk(keys) + self.Wq(query.unsqueeze(-2))))\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "        if attn_mask is not None:\n",
    "            # attn_mask is a matrix of 0/1 element,\n",
    "            # 1 means to mask logits while 0 means do nothing\n",
    "            # here we add -inf to the element while mask == 1\n",
    "            attn_mask = (attn_mask.unsqueeze(-1)) * -1e16\n",
    "            scores += attn_mask\n",
    "        scores = F.softmax(scores, dim=-2)\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "        context_vector = torch.mul(scores, values).sum(dim=-2)\n",
    "        # context_vector.shape = [batch size, hidden_dim]\n",
    "        return context_vector, scores\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.529323Z",
     "iopub.execute_input": "2023-12-15T07:10:48.529668Z",
     "iopub.status.idle": "2023-12-15T07:10:48.539967Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.529642Z",
     "shell.execute_reply": "2023-12-15T07:10:48.539034Z"
    },
    "trusted": true
   },
   "execution_count": 24,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        vocab_size, \n",
    "        embedding_dim=256, \n",
    "        hidden_dim=1024, \n",
    "        num_layers=1,\n",
    "        ):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim + hidden_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        self.dropout = nn.Dropout(0.6)\n",
    "        self.attention = BahdanauAttention(hidden_dim)\n",
    "        \n",
    "    def forward(self, decoder_input, hidden, encoder_outputs, attn_mask=None):\n",
    "        # decoder_input.shape = [batch size, 1]\n",
    "        assert len(decoder_input.shape) == 2 and decoder_input.shape[-1] == 1, f\"decoder_input.shape = {decoder_input.shape}\"\n",
    "        # hidden.shape = [batch size, hidden_dim]\n",
    "        assert len(hidden.shape) == 2, f\"hidden.shape = {hidden.shape}\"\n",
    "        # encoder_outputs.shape = [batch size, sequence length, hidden_dim]\n",
    "        assert len(encoder_outputs.shape) == 3, f\"encoder_outputs.shape = {encoder_outputs.shape}\"\n",
    "        \n",
    "        context_vector, attention_score = self.attention(\n",
    "            query=hidden, keys=encoder_outputs, values=encoder_outputs, attn_mask=attn_mask)\n",
    "        # context_vector.shape = [batch size, hidden_dim]\n",
    "        embeds = self.embedding(decoder_input)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim]\n",
    "        # concatenate\n",
    "        embeds = torch.cat([context_vector.unsqueeze(-2), embeds], dim=-1)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim + hidden_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, 1, hidden_dim]\n",
    "        logits = self.fc(self.dropout(seq_output))\n",
    "        # logits.shape = [batch size, 1, vocab size]\n",
    "        return logits, hidden, attention_score\n",
    "    \n",
    "    "
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.541222Z",
     "iopub.execute_input": "2023-12-15T07:10:48.542010Z",
     "iopub.status.idle": "2023-12-15T07:10:48.555424Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.541974Z",
     "shell.execute_reply": "2023-12-15T07:10:48.554562Z"
    },
    "trusted": true
   },
   "execution_count": 25,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class Sequence2Sequence(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        src_vocab_size, #输入词典大小\n",
    "        trg_vocab_size, #输出词典大小\n",
    "        encoder_embedding_dim=256, \n",
    "        encoder_hidden_dim=1024, \n",
    "        encoder_num_layers=1,\n",
    "        decoder_embedding_dim=256, \n",
    "        decoder_hidden_dim=1024, \n",
    "        decoder_num_layers=1,\n",
    "        bos_idx=1,\n",
    "        eos_idx=3,\n",
    "        max_length=512,\n",
    "        ):\n",
    "        super(Sequence2Sequence, self).__init__()\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.max_length = max_length\n",
    "        self.encoder = Encoder(\n",
    "            src_vocab_size, \n",
    "            embedding_dim=encoder_embedding_dim, \n",
    "            hidden_dim=encoder_hidden_dim,\n",
    "            num_layers=encoder_num_layers,\n",
    "            )\n",
    "        self.decoder = Decoder(\n",
    "            trg_vocab_size, \n",
    "            embedding_dim=decoder_embedding_dim, \n",
    "            hidden_dim=decoder_hidden_dim,\n",
    "            num_layers=decoder_num_layers,\n",
    "            )\n",
    "        \n",
    "    def forward(self, *, encoder_inputs, decoder_inputs, attn_mask=None):\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "        # decoding with teacher forcing\n",
    "        bs, seq_len = decoder_inputs.shape\n",
    "        logits_list = []\n",
    "        scores_list = []\n",
    "        for i in range(seq_len):\n",
    "            # 每次迭代生成一个时间步的预测，存储在 logits_list 中，并且记录注意力分数（如果有的话）在 scores_list 中，最后将预测的logits和注意力分数拼接并返回。\n",
    "            logits, hidden, score = self.decoder(\n",
    "                decoder_inputs[:, i:i+1], \n",
    "                hidden[-1], # the hidden state of the last layer\n",
    "                encoder_outputs, \n",
    "                attn_mask=attn_mask\n",
    "                )\n",
    "            logits_list.append(logits)\n",
    "            scores_list.append(score)\n",
    "        \n",
    "        return torch.cat(logits_list, dim=-2), torch.cat(scores_list, dim=-1)\n",
    "    \n",
    "    @torch.no_grad()\n",
    "    def infer(self, encoder_input, attn_mask=None):\n",
    "        #infer用于预测\n",
    "        # encoder_input.shape = [1, sequence length]\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_input)\n",
    "        \n",
    "        # decoding\n",
    "        decoder_input = torch.Tensor([self.bos_idx]).reshape(1, 1).to(dtype=torch.int64) #shape为[1,1]，内容为开始标记\n",
    "        decoder_pred = None\n",
    "        pred_list = []\n",
    "        score_list = []\n",
    "        # 从开始标记 bos_idx 开始，迭代地生成序列，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        for _ in range(self.max_length):\n",
    "            logits, hidden, score = self.decoder(\n",
    "                decoder_input, \n",
    "                hidden[-1], \n",
    "                encoder_outputs, \n",
    "                attn_mask=attn_mask\n",
    "                )\n",
    "            # using greedy search\n",
    "            decoder_pred = logits.argmax(dim=-1)\n",
    "            decoder_input = decoder_pred\n",
    "            pred_list.append(decoder_pred.reshape(-1).item())\n",
    "            score_list.append(score)\n",
    "            \n",
    "            # stop at eos token\n",
    "            if decoder_pred == self.eos_idx:\n",
    "                break\n",
    "            \n",
    "        # return\n",
    "        return pred_list, torch.cat(score_list, dim=-1)\n",
    "\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.556843Z",
     "iopub.execute_input": "2023-12-15T07:10:48.557362Z",
     "iopub.status.idle": "2023-12-15T07:10:48.573347Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.557328Z",
     "shell.execute_reply": "2023-12-15T07:10:48.572370Z"
    },
    "trusted": true
   },
   "execution_count": 26,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 训练"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 损失函数"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "def cross_entropy_with_padding(logits, labels, padding_mask=None):\n",
    "    # logits.shape = [batch size, sequence length, num of classes]\n",
    "    # labels.shape = [batch size, sequence length]\n",
    "    # padding_mask.shape = [batch size, sequence length]\n",
    "    bs, seq_len, nc = logits.shape\n",
    "    loss = F.cross_entropy(logits.reshape(bs * seq_len, nc), labels.reshape(-1), reduce=False)\n",
    "    if padding_mask is None:#如果没有padding_mask，就直接求平均\n",
    "        loss = loss.mean()\n",
    "    else:\n",
    "        # 如果提供了 padding_mask，则将填充部分的损失去除后计算有效损失的均值。首先，通过将 padding_mask reshape 成一维张量，并取 1 减去得到填充掩码。这样填充部分的掩码值变为 1，非填充部分变为 0。将损失张量与填充掩码相乘，这样填充部分的损失就会变为 0。然后，计算非填充部分的损失和（sum）以及非填充部分的掩码数量（sum）作为有效损失的均值计算。\n",
    "        padding_mask = 1 - padding_mask.reshape(-1)\n",
    "        loss = torch.mul(loss, padding_mask).sum() / padding_mask.sum()\n",
    "\n",
    "    return loss\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.574711Z",
     "iopub.execute_input": "2023-12-15T07:10:48.575073Z",
     "iopub.status.idle": "2023-12-15T07:10:48.588411Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.575041Z",
     "shell.execute_reply": "2023-12-15T07:10:48.587444Z"
    },
    "trusted": true
   },
   "execution_count": 27,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Callback"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "\n",
    "class TensorBoardCallback:\n",
    "    def __init__(self, log_dir, flush_secs=10):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            log_dir (str): dir to write log.\n",
    "            flush_secs (int, optional): write to dsk each flush_secs seconds. Defaults to 10.\n",
    "        \"\"\"\n",
    "        self.writer = SummaryWriter(log_dir=log_dir, flush_secs=flush_secs)\n",
    "\n",
    "    def draw_model(self, model, input_shape):\n",
    "        self.writer.add_graph(model, input_to_model=torch.randn(input_shape))\n",
    "        \n",
    "    def add_loss_scalars(self, step, loss, val_loss):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/loss\", \n",
    "            tag_scalar_dict={\"loss\": loss, \"val_loss\": val_loss},\n",
    "            global_step=step,\n",
    "            )\n",
    "        \n",
    "    def add_acc_scalars(self, step, acc, val_acc):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/accuracy\",\n",
    "            tag_scalar_dict={\"accuracy\": acc, \"val_accuracy\": val_acc},\n",
    "            global_step=step,\n",
    "        )\n",
    "        \n",
    "    def add_lr_scalars(self, step, learning_rate):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/learning_rate\",\n",
    "            tag_scalar_dict={\"learning_rate\": learning_rate},\n",
    "            global_step=step,\n",
    "            \n",
    "        )\n",
    "    \n",
    "    def __call__(self, step, **kwargs):\n",
    "        # add loss\n",
    "        loss = kwargs.pop(\"loss\", None)\n",
    "        val_loss = kwargs.pop(\"val_loss\", None)\n",
    "        if loss is not None and val_loss is not None:\n",
    "            self.add_loss_scalars(step, loss, val_loss)\n",
    "        # add acc\n",
    "        acc = kwargs.pop(\"acc\", None)\n",
    "        val_acc = kwargs.pop(\"val_acc\", None)\n",
    "        if acc is not None and val_acc is not None:\n",
    "            self.add_acc_scalars(step, acc, val_acc)\n",
    "        # add lr\n",
    "        learning_rate = kwargs.pop(\"lr\", None)\n",
    "        if learning_rate is not None:\n",
    "            self.add_lr_scalars(step, learning_rate)\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.589669Z",
     "iopub.execute_input": "2023-12-15T07:10:48.589949Z",
     "iopub.status.idle": "2023-12-15T07:10:48.605357Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.589924Z",
     "shell.execute_reply": "2023-12-15T07:10:48.604526Z"
    },
    "trusted": true
   },
   "execution_count": 28,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = - np.inf\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.608626Z",
     "iopub.execute_input": "2023-12-15T07:10:48.608907Z",
     "iopub.status.idle": "2023-12-15T07:10:48.621234Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.608883Z",
     "shell.execute_reply": "2023-12-15T07:10:48.620377Z"
    },
    "trusted": true
   },
   "execution_count": 29,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = - np.inf\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.622141Z",
     "iopub.execute_input": "2023-12-15T07:10:48.622418Z",
     "iopub.status.idle": "2023-12-15T07:10:48.636320Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.622394Z",
     "shell.execute_reply": "2023-12-15T07:10:48.635527Z"
    },
    "trusted": true
   },
   "execution_count": 30,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### training & valuating"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for batch in dataloader:\n",
    "        encoder_inputs = batch[\"encoder_inputs\"]\n",
    "        encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "        decoder_inputs = batch[\"decoder_inputs\"]\n",
    "        decoder_labels = batch[\"decoder_labels\"]\n",
    "        decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "        \n",
    "        # 前向计算\n",
    "        logits, _ = model(\n",
    "            encoder_inputs=encoder_inputs, \n",
    "            decoder_inputs=decoder_inputs, \n",
    "            attn_mask=encoder_inputs_mask\n",
    "            )\n",
    "        loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)         # 验证集损失\n",
    "        loss_list.append(loss.cpu().item())\n",
    "        \n",
    "    return np.mean(loss_list)\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.637391Z",
     "iopub.execute_input": "2023-12-15T07:10:48.637711Z",
     "iopub.status.idle": "2023-12-15T07:10:48.651242Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.637685Z",
     "shell.execute_reply": "2023-12-15T07:10:48.650379Z"
    },
    "trusted": true
   },
   "execution_count": 31,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 1\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for batch in train_loader:\n",
    "                encoder_inputs = batch[\"encoder_inputs\"]\n",
    "                encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "                decoder_inputs = batch[\"decoder_inputs\"]\n",
    "                decoder_labels = batch[\"decoder_labels\"]\n",
    "                decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "                \n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # 前向计算\n",
    "                logits, _ = model(\n",
    "                    encoder_inputs=encoder_inputs, \n",
    "                    decoder_inputs=decoder_inputs, \n",
    "                    attn_mask=encoder_inputs_mask\n",
    "                    )\n",
    "                loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "                \n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                \n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    "            \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "                    \n",
    "                    # 1. 使用 tensorboard 可视化\n",
    "                    if tensorboard_callback is not None:\n",
    "                        tensorboard_callback(\n",
    "                            global_step, \n",
    "                            loss=loss, val_loss=val_loss,\n",
    "                            lr=optimizer.param_groups[0][\"lr\"],\n",
    "                            )\n",
    "                \n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), metric=-val_loss)\n",
    "\n",
    "                    # 3. 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "            pbar.set_postfix({\"epoch\": epoch_id, \"loss\": loss, \"val_loss\": val_loss})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 20\n",
    "batch_size = 64\n",
    "\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True, collate_fn=collate_fct)\n",
    "test_dl = DataLoader(test_ds, batch_size=batch_size, shuffle=False, collate_fn=collate_fct)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = cross_entropy_with_padding\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# 1. tensorboard 可视化\n",
    "if not os.path.exists(\"runs\"):\n",
    "    os.mkdir(\"runs\")\n",
    "exp_name = \"translate-seq2seq\"\n",
    "tensorboard_callback = TensorBoardCallback(f\"runs/{exp_name}\")\n",
    "# tensorboard_callback.draw_model(model, [1, MAX_LENGTH])\n",
    "# 2. save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\n",
    "    f\"checkpoints/{exp_name}\", save_step=200, save_best_only=True)\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=5)\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "record = training(\n",
    "    model, \n",
    "    train_dl, \n",
    "    test_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    tensorboard_callback=tensorboard_callback,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=200\n",
    "    )"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:10:48.652448Z",
     "iopub.execute_input": "2023-12-15T07:10:48.652787Z",
     "iopub.status.idle": "2023-12-15T07:25:59.933077Z",
     "shell.execute_reply.started": "2023-12-15T07:10:48.652762Z",
     "shell.execute_reply": "2023-12-15T07:25:59.932128Z"
    },
    "trusted": true
   },
   "execution_count": 32,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/33500 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a77ae58552e648a38eee1c8a63f8a4a5"
      }
     },
     "metadata": {}
    },
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n  warnings.warn(warning.format(ret))\n",
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": "Early stop at epoch 4 / global_step 8000\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"]], [i[\"loss\"] for i in record[\"train\"]], label=\"train\")\n",
    "plt.plot([i[\"step\"] for i in record[\"val\"]], [i[\"loss\"] for i in record[\"val\"]], label=\"val\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:25:59.934489Z",
     "iopub.execute_input": "2023-12-15T07:25:59.934824Z",
     "iopub.status.idle": "2023-12-15T07:26:00.223805Z",
     "shell.execute_reply.started": "2023-12-15T07:25:59.934796Z",
     "shell.execute_reply": "2023-12-15T07:26:00.223016Z"
    },
    "trusted": true
   },
   "execution_count": 33,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXSElEQVR4nO3dd3gU1cIG8He2JiGVhFQCoQdC712UqoBc5dpAxYooKqAXxYKCiqDez2vHgoqKgCCiKFIivffQW+gEkhBCettyvj82u8lmN2XDJpNM3t/z8GQzMzt7DgvZN6dKQggBIiIiIjdQyV0AIiIiUg4GCyIiInIbBgsiIiJyGwYLIiIichsGCyIiInIbBgsiIiJyGwYLIiIichsGCyIiInIbTXW/oNlsxpUrV+Dj4wNJkqr75YmIiKgShBDIzMxEeHg4VKrS2yWqPVhcuXIFkZGR1f2yRERE5AaXLl1Cw4YNSz1f7cHCx8cHgKVgvr6+bruvwWDA2rVrMWTIEGi1Wrfdt6ZQev0A5ddR6fUDlF9H1q/2U3odq7J+GRkZiIyMtH2Ol6bag4W1+8PX19ftwcLLywu+vr6K/cei5PoByq+j0usHKL+OrF/tp/Q6Vkf9yhvGwMGbRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNooJFv/7Jx7LzqmQmJEnd1GIiIjqrGrf3bSqLN13GdeyVLiRbUBkoNylISIiqpsU02JR3jauREREVPUUEyysBITcRSAiIqqzFBMsrO0VgrmCiIhINooJFmBPCBERkeyUEyyIiIhIdooJFuwKISIikp9yggVnhRAREclOMcHCirNCiIiI5KOYYMGuECIiIvkpJ1iwJ4SIiEh2igkWVmywICIiko9igkVRVwijBRERkVwUEyysfSGMFURERPJRTLDgEAsiIiL5KSZY2LDJgoiISDaKCRbWWSHMFURERPJRTrBgZwgREZHsFBMsrDgrhIiISD6KCRbsCiEiIpKfcoKF3AUgIiIi5QQLK/aEEBERyUcxwaKoK4TJgoiISC6KCRbsDCEiIpKfgoKFBbtCiIiI5KOYYMFt04mIiOSnnGBR+JUtFkRERPJRTrBgiwUREZHsFBMsrDgrhIiISD6KCRbWvULYFUJERCQf5QQLdoUQERHJTjHBwooNFkRERPJRTLDgrBAiIiL5KSZYsC+EiIhIfsoJFoU4K4SIiEg+igkWtvYK5goiIiLZKCdYsCeEiIhIdooJFlZssCAiIpKPYoKFtcVCcFoIERGRbJQTLMC+ECIiIrkpJlhYsb2CiIhIPooJFkVdIfKWg4iIqC5TTrAo/MpcQUREJB/FBAsOsSAiIpKfcoJFIc4KISIiko9iggVnhRAREclPMcHChg0WREREslFMsLDNCpG3GERERHWacoKF3AUgIiIi5QQLK47dJCIiko9igoVU2Bci2BlCREQkG+UEC7kLQERERK4FC5PJhOnTp6NJkybw9PREs2bN8Pbbb9eotSNqUFGIiIjqHI0rF7/33nuYO3cufvjhB8TExGDv3r149NFH4efnh+eff76qylghEpssiIiIZOdSsNi+fTtGjRqF4cOHAwCioqKwaNEi7N69u0oKVxlssCAiIpKPS8Gid+/e+Prrr3Hq1Cm0bNkSBw8exNatW/Hhhx+W+pz8/Hzk5+fbvs/IyAAAGAwGGAyGShbbkbU7xmg0uvW+NYW1Tkqsm5XS66j0+gHKryPrV/spvY5VWb+K3lMSLgyQMJvNePXVV/H+++9DrVbDZDJh1qxZeOWVV0p9zowZMzBz5kyH4wsXLoSXl1dFX7pcnxxR40ymhEdbmtAxkO0WRERE7pSTk4MxY8YgPT0dvr6+pV7nUovFkiVL8PPPP2PhwoWIiYlBXFwcJk+ejPDwcIwbN87pc1555RW88MILtu8zMjIQGRmJIUOGlFkwV/2UsAvITEe79u1xR4cIt923pjAYDIiNjcXgwYOh1WrlLk6VUHodlV4/QPl1ZP1qP6XXsSrrZ+1xKI9LwWLq1KmYNm0a7r//fgBAu3btcOHCBcyePbvUYKHX66HX6x2Oa7Vat1ZapbJMcFGr1Ir8x2Ll7r+3mkjpdVR6/QDl15H1q/2UXseqqF9F7+fSdNOcnBzbB7iVWq2G2Wx25TZERESkUC61WIwcORKzZs1Co0aNEBMTgwMHDuDDDz/EY489VlXlqzBuQkZERCQ/l4LFp59+iunTp+OZZ55BcnIywsPD8dRTT+GNN96oqvJVmHUZi5q0WBcREVFd41Kw8PHxwUcffYSPPvqoiopDREREtZly9gqxbUJGREREclFOsJC7AERERKScYGHFIRZERETyUU6w4KwQIiIi2SknWBAREZHsFBMsJFuTBdssiIiI5KKcYMGuECIiItkpJlgQERGR/BQTLIpW3pS1GERERHWaYoIFERERyU8xwaJojAWbLIiIiOSinGBR2BnCrhAiIiL5KCZYEBERkfyUEyw43ZSIiEh2igkWnBVCREQkP8UECyIiIpKfYoKFZNs3nU0WREREclFMsCAiIiL5KSZYcLopERGR/JQTLDgrhIiISHaKCRZEREQkP8UEC043JSIikp9ygkVhXwj3CiEiIpKPYoIFERERyU9xwYJdIURERPJRXrCQuwBERER1mGKCRdHKm0RERCQX5QQL6wP2hRAREclGMcGCiIiI5KeYYFE03ZSIiIjkophgYcWeECIiIvkoJlhw7CYREZH8lBMsuAkZERGR7BQTLKwE+0KIiIhko5hgIbEzhIiISHaKCRZgVwgREZHslBMsiIiISHaKCRbWjhAOsSAiIpKPYoIFERERyU8xwYKbkBEREclPOcGisDOE002JiIjko5hgYcVYQUREJB/FBAt2hRAREclPOcGi8Ct7QoiIiOSjmGBBRERE8lNMsCjahIxNFkRERHJRTLCwYlcIERGRfJQTLDh6k4iISHaKCRYcvElERCQ/xQQLIiIikp9iggV7QoiIiOSnmGBhxSW9iYiI5KOYYCGBTRZERERyU06wsK1jQURERHJRTLCwYk8IERGRfBQTLNgRQkREJD/lBAt2hRAREclOMcHCirNCiIiI5KOgYMHOECIiIrkpKFhYsL2CiIhIPooJFraVN5ksiIiIZKOcYCF3AYiIiEg5wcJKsMmCiIhINooJFtyEjIiISH4uB4uEhAQ8+OCDCAwMhKenJ9q1a4e9e/dWRdkqhbNNiYiI5KNx5eIbN26gT58+uPXWW7Fq1So0aNAAp0+fRkBAQFWVr8Ksm5AxVxAREcnHpWDx3nvvITIyEt9//73tWJMmTdxeqMpgVwgREZH8XAoWK1aswNChQ3HPPfdg06ZNiIiIwDPPPIMnn3yy1Ofk5+cjPz/f9n1GRgYAwGAwwGAwVLLYjsxmMwDAZDK59b41hbVOSqybldLrqPT6AcqvI+tX+ym9jlVZv4reUxIurIHt4eEBAHjhhRdwzz33YM+ePZg0aRK+/PJLjBs3zulzZsyYgZkzZzocX7hwIby8vCr60uX68bQK+1JU6NHAjDHNzW67LxEREQE5OTkYM2YM0tPT4evrW+p1LgULnU6Hrl27Yvv27bZjzz//PPbs2YMdO3Y4fY6zFovIyEikpKSUWTBXtZi+1vb49NtD3HbfmsJgMCA2NhaDBw+GVquVuzhVQul1VHr9AOXXkfWr/ZRex6qsX0ZGBoKCgsoNFi51hYSFhaFNmzZ2x1q3bo1ly5aV+hy9Xg+9Xu9wXKvVVtmbqsR/LFZV+fdWUyi9jkqvH6D8OrJ+tZ/S61gV9avo/VyabtqnTx+cPHnS7tipU6fQuHFjV25DRERECuVSsJgyZQp27tyJd999F/Hx8Vi4cCG+/vprTJw4sarKV2ER/h5yF4GIiKjOcylYdOvWDcuXL8eiRYvQtm1bvP322/joo48wduzYqipfhT3UsxEAYFSHMJlLQkREVHe5NMYCAEaMGIERI0ZURVluilplWcjCZOYSWURERHJRzF4haonBgoiISG7KCRaFLRZGBgsiIiLZKCZYaAqDhZm7kBEREclGMcHC2mKRZ+Cqm0RERHJRTLA4kZgJANh25rrMJSEiIqq7FBMslsddkbsIREREdZ5igsXjfaLkLgIREVGdp5hg0S0qAADQuL77dkwlIiIi1ygmWKg5K4SIiEh2igkW1umml27kylwSIiKiuksxwaLAWDTNNN9okrEkREREdZdigkV2QVGYyDdyLQsiIiI5KCZYWMdYEBERkXwUEyw0xYKFYIMFERGRLBQTLHSaoqqYODOEiIhIFooJFh0i/GyPuXU6ERGRPBQTLFQqCSpYAgXXsiAiIpKHYoIFAJhhGWeRlmOQuSRERER1k6KChdUn607LXQQiIqI6SZHB4ko6V98kIiKSgyKDhcHE+aZERERyUGSwOJKQIXcRiIiI6iRFBgsiIiKSB4MFERERuY1ig0WegTucEhERVTfFBovPN8TLXQQiIqI6R7HB4tP18RBcgZOIiKhaKTZYAMDRK5wdQkREVJ0UFSzGNrMfV8FxFkRERNVLUcEi2NO+64MdIURERNVLUcFCkrsAREREdZyigoWKyYKIiEhWigoW4V7233NSCBERUfVSVLBQK6o2REREtQ8/iomIiMhtFB0sLqbmyF0EIiKiOkXRweI/Sw/KXQQiIqI6RdHBgoiIiKoXgwURERG5DYMFERERuQ2DBREREbmN4oLF8gk95S4CERFRnaW4YNE2wlfuIhAREdVZigsWJZ1LyZa7CERERHWG4oPFrf/diA9jT8ldDCIiojpB8cECAD5Zd1ruIhAREdUJdSJYEBERUfVQZLDQaRRZLSIiohpPkZ/Aozs3lLsIREREdZIig4VakbUiIiKq+RT5ESxBkrsIREREdZIig4XRbJa7CERERHWSIoOFWsUWCyIiIjkoMliE+Hg4HDt+NQNCCBlKQ0REVHcoMlhIThosbv94C97661j1F4aIiKgOUWSwqF9P7/T499vOV29BiIiI6hhFBot/d+E6FkRERHJQZLDQaVRYPbmf3MUgIiKqcxQZLAAgOtRX7iIQERHVOYoNFqW5npUvdxGIiIgUq84Fi63xKXIXgYiISLHqXLCYtDgO87achdnMNS2IiIjcTdHBonuT+k6Pv7PyOJbtv1zNpSEiIlI+RQeLnk0DSz0Xn5xVjSUhIiKqGxQdLJ6+pVnpJ7mdCBERkdvdVLCYM2cOJEnC5MmT3VQc9/LUqeUuAhERUZ1S6WCxZ88efPXVV2jfvr07y1NtJDZZEBERuV2lgkVWVhbGjh2Lb775BgEBAe4uU7VwtlEZERER3RxNZZ40ceJEDB8+HIMGDcI777xT5rX5+fnIzy9alCojIwMAYDAYYDAYKvPyTlnvVdF7Gowmt75+VXO1frWR0uuo9PoByq8j61f7Kb2OVVm/it5TEkK4tKDD4sWLMWvWLOzZswceHh4YMGAAOnbsiI8++sjp9TNmzMDMmTMdji9cuBBeXl6uvHSlTNpRenb6uJexyl+fiIhICXJycjBmzBikp6fD17f0bTNcChaXLl1C165dERsbaxtbUV6wcNZiERkZiZSUlDIL5iqDwYDY2FgMHjwYWq3WdrzF9LWlPmf3KwMQ4KUDAAghINXg/pHS6qckSq+j0usHKL+OrF/tp/Q6VmX9MjIyEBQUVG6wcKkrZN++fUhOTkbnzp1tx0wmEzZv3ozPPvsM+fn5UKvtZ2Lo9Xro9XqHe2m12ip5U0vet2WIN04lOV+z4pH5+/H3pH7YFp+C5xYdwOy722FoTKjby+ROVfX3VpMovY5Krx+g/DqyfrWf0utYFfWr6P1cGrw5cOBAHD58GHFxcbY/Xbt2xdixYxEXF+cQKmqCv58vffv0Y1ct4z3GztuF1OwCPPXTvuoqFhERkSK51GLh4+ODtm3b2h2rV68eAgMDHY7XFBp12dnpanpuNZWEiIhI+RS98qbVI72jSj3Xa/b66isIERGRwlVqumlxGzdudEMxqtabI9tg/vbzcheDiIhI8epEi0VNnu1BRESkJHUiWBAREVH1YLAowWAyy10EIiKiWqvOBItuURXb06TAyGBBRERUWXUmWMx7uFuFrnNpfXMiIiKyU2eChZ9XxVYMc3HrFCIiIiqmzgQLAHisT5Nyr2GsICIiqrw6FSz+M7Rludecu5aN9BwD8gymaigRERGRstz0Alm1iZeu/Op+tiEesceSUE+nxtG3hlVDqYiIiJSjTrVYVETssSQAQHaBCem5BplLQ0REVLswWJRh1spjcheBiIioVqlzweL14a0RiuuYplkIT+SVee2SvZdLPWc2c5gnERFRSXUuWLQK8cZPujmYoPkLY9Tl72x6JS3XYQrq0SvpaDdjDb7ZfLaqiklERFQr1blg0SbcD/NMdwAAntL8BT0Kyry+95z1ePuv43bHXl1+BNkFJsz6+3gpzyIiIqqb6lywCPTWo8/oibgsghAspeE+9YZyn/PdtnPs+iAiIqqAOhcsACDE3xdfGkcCACZo/oQO5c/+WHssiatyEhERlaNOBgtJkrDUdAsSRQDCpVSMVm8u9zkTFuzDR/+ctjy/qgtIRERUS9XJYNG5kT/yocNXxhEAgGfUK6CBsdznfbzuNPKNJpjZckFERORUnQwWGrWl2otMt+Ga8EWk6hruUm+t0HO/23oehy6nV2XxiIiIaq06GSys8qDHN8bhAIBn1H9AjfL3B5m7Md7u+4//Oc2xF0RERIXqbLAY0T4MALDANBipwhtNVEkYodpR7vMy8uy7TP73zymsLVwGnIiIqK6rs8Hi4/s74ZkBzZADDyTHPAEAeFbzB1Qwu3yvhBu57i4eERFRrVRng4VaJeGlYdE4Pet2RN/5ItKFF1qoEnC7arfL9/p8QzyMJtcDCRERkdLU2WBhpVWrAA9ffG+ybJH+rGY5JBdbLa5nF2Dxnks4kZiBeVvO4scd5/Hkj3uRZyh/zAYREZGSaOQuQE3xnXEYHlevQmvVJQxW7cNaczeXnh+fnIXXfz9id2zR7ot4tE8TdxaTiIioRqvzLRZWGfDGj6bBAIDnNMsB3PxMj/Tc8lf0JCIiUhIGi2LmGe9AttCjneo8BqjiXHqusymnnIVKRER1DYNFMTfgiwWmQQCASS62WpicBYvij4XAD9vP49DltJsrJBERUQ3GYFHCPONw5AktOqni0Vd1pPwnFFqw86LjwWJh469DV/HmiqO487Nt7igmERFRjcRgUcI1+GOR6TYA1rEWlVe8xeJUUuZN3YuIiKg2YLBw4kvjSOQLDXqoTqCHdLzS9+EYCyIiqmsYLJxIQn0sMQ0AADyv+a3S9/lsQzxuZBcAsN9q/eVfD2H5gcsu36+Ay2IQEVENx2BRii+NI2EQavRRH0Vn6VSl79Pp7ViHY7/svYQpvxzEtvgUGCq4Yuey/QmYuluDRXsuVbosREREVY3BohQJaIBlpn4AgOdvcqxFacbO24Ue766r0LXTlh8FALyxovJdM0RERFWNwaLQnLvboVtUgN2xL0yjYBQqDFAfRHvpzM29gCQ5PZxa2FVSloq2ahAREcmNwaLQ/d0bYemE3vDz1NqOXRQhuBI5AgAwQ/sDtDCW9vRyLdtX9pgKIQRyChzv//22c2jx2qpKvy4REVF1YrAooeQKmo1GvwN4+KGzKh6va36q1D2jpq1EQlrZW6uP/2kf2ryxBlN+icOBizfw6Pe7sT0+BTP/PFap1yQiIpIDg0UJXz3U1f5AQGPg7m8AAOM0sbhLtcXtr/nrvsuIPZYEAFh+IAF3fbEdG05ew5h5u9z+WkRERFWJwaKEXs0CcebdO/DWqBj89Hh3y8GWQ4FbXgYAvKv9Fq2lC259zf8sPejS9Z+sO81xF0REVCMxWDihVkl4uFcU+rVoUHTwlmlA88HwlArwpfZ/8EWWbOX7MPYUftpRFG6EEPjvmpNYupdTUYmISF4MFhWlUgF3f42L5gZorErGR9ovIEG+VoPTyUXBZv/FNHy2IR5Tfz2Ef44lwWx2XPJze3wKNp+6Vu59s/ONmLflLC6l5ri1vEREVDcwWLjCqz6eNkxBntDiNnUcnlP/LltRrmXm2x6n5xZNWX3ix7249f82It9YtExnvtGEMfN24eHvdiMjz+Bwr+UHLqPve+vx9IJ9ePyHPXhn5XEM/8T9Y0mIiEj5GCxcdFRE4TXD4wCAyZplGKCKk6UcJ5MykGdwvsb3hes52HU21fZ9Zp7R6WOrKb8cxOUbuVh1JBE7C5+X4eQ6IiKi8jBYuOifF27BsZAR+Mk4CCpJ4GPtZ4iUkqq9HJdScxE9fTW+3XrO6fninSELdznZ0p2IiKgKMFi4qHmwNybe2gxvGx/CAXNz+Ek5+FL7ETyQX/6Tq8Dbfx2DBMdVPc3F1uNIy3Hs/qiIP+IScC4l2/Z9QloutsWnOKz1QUREZMVgUQkDo0MQUt8Xnwa9gRThixjVBczSfgf7doLq8+j8PQ7HhBD4+/BVfLj2pN1q4q6EgkmL43Drfzfiy01nkJCWiz5z1mPsvF1Yc7T6W2iIiKh20MhdgNrIU6fGpv/ciuvZBXhu9nNYoH0Xo9VbcMDcHAtMg+UuHgDAbAae+Xk/AMDfq2iZ8sT0PAT7eECnqXimnLPqBOZtKepy2XAiGcPahrqvsEREpBhssagklUqCJAE7zDGYY3wAAPCG5kd0kk7LXDKLJ37ca3tcvCvk31/uwL8+3+by/VKy5OnqISKi2oXB4iaoCvsYvjENx2G/AdBJJnyl/wRBSJe5ZGU7djXD9vhEYkYZVzonZOryISKimo/B4iaobGMXJCTf+n9AUCsE4zq+0n2IALj+gS2HYR+5vl4Fx24SEVFpGCxuglRsVGSbJhHAfQsAvS+6qE5jhW46Wkk1e5rnpgqsxElEROQKBouboCo228IsADRoCTy+FufNIYhUXcNvujcxVOU4Y6OmGPfd7ko972xKNvIMJsQeS0J2PhfSIiKiIgwWN8FLVzSpJrCezvIguDVOjPwdW0xtUU/Kx1e6/2GSepms+4o488P285V+7r4LN/DyskN48se9eObn/fhf7Cnc9n8bcSUt130FJCKiWonB4iaoVRIOvjEE+6cPhodWbTs+rFsbnBr0Pb4zDgMATNEuw+faT+CFPLmK6uDNFUdv6vl/xF0BYOlO+XjdaZy9lo0ZN3lPIiKq/RgsbpKflxb1ra0VxWh1OrxlfBhTDeNRINS4Q70by3Qz0FBS7riG7AJ2ixAR1XUMFlWkS+MAAMBS0wD8X/iHuCb80Fp1EX/oXkcP6bjMpasanC1CREQMFlUkJtwPy57uje3TbsMV3w64M/8dHDI3QaCUiQW6d/GgOlbuIrodgwURETFYVKEujQMQ7u8JALiKQNxT8Cb+MPWGVjLhHe33mKX5Flqw+wAA9p5PxYXUHLmLQUREN4nBohpYN/7Khw6TDBMxx3A/BCSM1azDQt07aC1dkLmE7lHZFTnjkzPx7y93YND/tgIALt+wbHg2b8tZdxavUkxmgdeWH8YfcQlyF4WIqFZgsKh2Er403QlpzC/IEJ7opjqFVfpX8In2UzSRrspduJtSVlfIkYR09Ht/Pf48eMXh3NErRauUHk6V8OKvh5GQlot3Vso/FuXPg1fw866LmLQ4zun5yzdysO9CavUWioioBmOwqAZOP29bDsVzPh9hhakXAOBO9Q7E6qZituYbhOF6tZbPXYrXM89gsjs34tOtuJSai+cWHXB4XvEVTOedVGP/xTS3lis+OcvlNTay843YHp+C5Myypwj3fW8DRs/d4XTPlYw8A9YeTUS+0eTkmUVOJWVi/I97cfyq82XgjySk4/Dlmr3/DBGRFYNFNXi8bxOnx+e/eB8in1yEO/LfxT+mTtBIZjyg2YBNHlMwXfMTAmv4ZmYOhOUDefyPexE9fTWipq3ErJXHcCO7wOHSq+m5SMtxPF4ZmXmGUs+lZOVj0Ieb0HvOepfu+ej8PRgzbxfmbjxToesPOfngf/T7PRj/0z58sPpkmc994OudWHssCaPnbnc4l280YcSnWzHys63I4XReIqoFGCyqQedGATgwfTAe6B5pd1ySJHRqFIBjIgpPGKbi7vwZ2GluDR2MeFyzCpv1k/GCZgl8UDsGNQoIvPTrIaw9lmQ79s2Wc9h1zr6rIDkjD71mr0fHtywzYyS4Zt3xJIz7bjde/vUQvtl8Fu1mrMW3W88BAAwmM8bO24n/rrF8mJ9JzqpUXXYXlvlGjmNoMZsd26Cc1WHfhRsAgHlbz+GH7ecx+KOtuJxddD4zz4AXlxzE9cLglVPg2LKRV1C0YmtWHoMFEdV8DBbVJKCeDqV9hD5R2KKxX7TE/QWvY03nuTCHdUI9KR/Pa37HFv0kPKX+Ex7Ir8YSu27P+RtYedhxnMjhhDS777u/u86l+775xxEM+2izrXvl8R/2YtOpa/hl7yXM+tsyDuPtv44BAGKPJWFb/HV8tiG+EjUo39K9l9B2xhpsP5Pi0vPeXHEU56/n4PuTagghcOF6Nj765zSW7b9cJeUkIpKLS8Fi9uzZ6NatG3x8fBAcHIx//etfOHmy7GZeKuKtVzs9/vqINmgSVK/wOwmX6/eCavwGPFUwBafMEfCXsvGKdhE266fgEfVq6OGeLoTqIpXRJjH+x73IKmMjs++2nsMPOy7gRGImVh4qf3BrgbH0PVncsWHa1F8PIafAhPE/7sPXm4u6SYqPEylLSr6EH3ZexC0fbLS1sjhzIjEDD327C3GX0262yERE1cqlYLFp0yZMnDgRO3fuRGxsLAwGA4YMGYLs7Ozyn0x49tYW6BYVgHfvaudwLrK+l+2xEAKQJDTtfz+GFbyHKQVP46K5AYKlNMzQ/ohN+ikYp15T6wKGM2uPJeGTdadLPf9WYUsEALy49GC59ys+5fW91Sew6kii7fuYN9fg2YX7AQC/H0jAhhPJtnPJmXm2acGluVRsnY2sfCPe/fuE7fvisWLN0UQcKiMQzPq7/DD+4Lzd2HI6pdI70BIRyUVT/iVFVq9ebff9/PnzERwcjH379qF///5uLZgS+XlpsXRCb6fn3h/dHj1nW7oIGgZYFtV6eVg0ft55Acvz+uGvgl64R70JEzW/I0K6jpnaHzBB8ye+MN6JX0y3ogDaaquHq8r7Zf5qesU3Z0vKKP3aq+m52HKqqIvC2cDLvw5dxbTbczD5lzgAwOA2IbiUmoMTiZkYFhOK89ezcSIx0+n9+72/odzynUzMxFM/7Sv3utLc9cU2jOoQjpSsmt3tRURUGpeCRUnp6ZaR8PXr13dLYeqyUD8P/PxED8RdSsPQmFDbcZ3G0qhkgAarNbfh1/z+toARLqXibe18PK1ZgS+Mo7DENKBGBwx36FHG+Ixesys28+N6VlFLT2yxgaarjyY6u7zCzGaB08nOQ0lFHbiYhgNunm5LRFSdKh0szGYzJk+ejD59+qBt27alXpefn4/8/KLfvjIyLHP1DQYDDIbSpwm6ynovd96zunVv7Ifujf1gNBaNBSj+y/6bnU0whnfE9D89kNrk32h/7U/8K+sXhEupeEf7vS1gLDXdUqMCxvIDNWvVyh1n3L/DrNlswui523DgUtVNETYYjTX+37cS/h+WhfWr/ZRex6qsX0XvKYnyOpZL8fTTT2PVqlXYunUrGjZsWOp1M2bMwMyZMx2OL1y4EF5eXk6eQcV9cEiNy9mWePFxL/vBh/89pEZytgH3qTfgGc0KhEqW6Y1XRCA+M/4Lv5r616iAQTfn+RgjQj2BenxLZZVtALw05XfxESlNTk4OxowZg/T0dPj6+pZ6XaWCxbPPPos//vgDmzdvRpMmzhd/snLWYhEZGYmUlJQyC+Yqg8GA2NhYDB48GFqtcn7yXrieg+krjuGJ3pHIOrPPrn7PLz6IVUctTfnP9YtAu6Tl6Jf0MzzzLb+RZwgvxJq74C9TT2w1t4OhnAaqqUNa4IO1pQ+kpJrh9NtD8PfhRExbfgTfPtwF3aIC5C6SjVL/H1ptPpmExxccxIh2IfjfvR3kLo7bKf39A5Rfx6qsX0ZGBoKCgsoNFi51hQgh8Nxzz2H58uXYuHFjuaECAPR6PfR6vcNxrVZbJW9qVd1XLs1D/bBofC8YDAb8fca+fm/9qx3UahXG9miMXs0CAXQEDNOAffORv/kj+OYkYrR6C0artyBDeGGtuStWmnqUGjL6tghGsK8npv56qHorSS7RarWYtMTyHo35dg9WPt8XrUN9oVLVnF+hlfb/0Gre9ksAgL8OJ+Gzscqrn5VS37/ilF7HqqhfRe/n0nTTiRMnYsGCBVi4cCF8fHyQmJiIxMRE5Oa6tg8DuUcDHz0+G9O5MFQU0noCPZ+G/j/HcU/+G/jeOBRJwh++Ug7+rd6M73UfYK9+Av6r/RIDVAfstm2XJOCerpE4N/sO+Hkq9z9cbfdbiUW1hn+yFZ+sZ0tTSSX3qyGi6uFSsJg7dy7S09MxYMAAhIWF2f788ssvVVU+qiyVCi8/9ShmGsehZ/5ndiHDrzBkzNd9gMPeE/GB5kuMVG1HWM5JID8TkiRBU4N++yV7LyxxXM/js/VVs9JobbX2aCKip6/GV5uKphyX1uu7/+IN3PnZVuw5z11qidzB5a4Qqj26RtXHmyPbYOafx7BHRGOPMRpvGR9CV+kUhqt34nb1boQY03CPZjPuwWZg4WeWJ9YLxjxTfZzWhOC8CMU5EYrzIhTnRQhy4SFvpcipujyQcNm+y/hgzUnMG9cVbSP8AAD/KVxMbfaqE3jqlmZ4b/UJ/LrvMlY+1xfBvvb/hu/9cgeMZoF7vtyB83OGl/ladfivmajCbmodC6pdfnisO8Z9txt7RDRM4T1x+5iOQFoccOwP4GoccP0MkJMCZCejE5LRSXPC4R6JIgDx5nDsMMdgm7ktDommMHPLGdmVtWx6VTGZBV5cEofOjQPwcK+om7pXRp4BPnqN06XRTWaBpIw8hPt72o4du5KBiABP+HlqbSuyTlp8AOteHOD0/tbF0uZuOoM3R8bYnTM62VSOiCqPwULh/tUxAu+vPon+LYNwS8sGuLtTBH47kID/DG2FEP96gH8fIKpP0RPy0oHrZ/DKvD/QoOAyolSJaCIlIkpKRICUhVDpBkLVN9BXfRRTsQQZwgvbzTHYam6Lbea2OCdCwd/rZFANf+Xv/n0c1zLz8eG9HSBJEtYcTcTvcVfwe9yVmwoWX246gzmrTqB/ywb48bHuDuef+mkv/jmejG/HdcXA1iHYdyEVo+fugI+HBodnDLVdZ6pAQLjpRlf+066UdceTEOStR4dIf7mLQtWAwULhAurpcGjGENuYif+7twPeGNkG/l4650/w8AMiOmON6jpSTQVAsfFvfshCEykRbVXn0Ed1BL1VR+En5WCYeg+GqfcAABJEILaZ2mKruS22m9siBX5VXUVC5T7vrmXmY/OpaxjePgweWucb5BX39eazAICnbmmK6FBfZOa5ZwGeOassLWObTxUtXLZkzyW0CPFGp0YB+Oe4ZU+Xb7eew8DWIbbvM0tsI28Wlu5ag0kgo5Qt5pMzK758fEnZ+UZsP8NxGK46ey0Lj/+wFwDK7WoiZWCwqAO06qKuCkmSSg8V5UiHN+JEc8SZmmOBaTBUMKOtdA59VUfQV3UYXVSnECFdx72aTbgXmwAAJ8yROCqicNocgVOiIU6LCFwWDSDYfeJW5Y2xSM7IQ/d31+H525rjhSGtAAB3z92GS6m5OJGYgdeGt7Fd99mGeDzYszFahvg4vdf+C2mIDvV1+tv/lbRcvLFXjUk71uK7R7ritugQ27msfCPOXstCuwi/MneD3RafgpeWWabTFv8gup5VACGE0z1gAOBiag7az1zrEDiK+/twIi7fyEGEv6fTMpxLycamk8m4v3sjh7D1KQfIVsqFYpv3Ud3AYEFO+XpokJpd9u6pZqhwSDTDIVMzfGEaBQ/ko5vqJPqojqCv6gjaqs4jWnUJ0bgEFPsZnSt0iBfhOC0a4rTZEjZOiYa4LBpwvEYlORtjYTIL/H4gAV2jAnDLBxsBAJ+sj7cFi0uplmni/xxPtgWLKUvisC3+On7edRFn3r0DRxLSMeLTrXhrVNG4hFeXH0bvZoE4meS4L8rrfxxDusFSlsfm78X5OcOx8WQymgZ545Hvd+NsSja+fLALhrUNdXguAKw8dBUTC3egLelkUiYe+tZ+t9f0XPtWE2eh4mq6/XT4vu9tQIivHrteHeSwC+2t/90IAEjNMeCFwS1RYDRDq7bU50RihtNyUdlupvcot8CEZ37eh8FtQjGmRyO3lYmqFoMFOTX3wS6Y8ksc7mgXhg9jTzmcH94uDO0b+mHl4as4dNmyP0Ye9Nhibo8t5vYAgPrIQDfVSbSQLqOFKgEtpctoKl2Bp1SAdtJ5tMN5u8CRJ7Q4J0JxUYTgvAjBRRGCC4WPr4pAmFB+c31dJUmWnVXzDCZbP/Yvey7h1eWHHa5dezQRg1oXtSQU/8F/9Irlw9NkFth19jru+3onAOCNP47a3ePIlXR8v+28w71TsuzD6O5zqXjk+z12xz6MPYnezQPho9dg5eGrdudKhorsfPugsDU+xe77DjPXOpShpNeWH3E4lpRhWQ34zs+2OX3OnnOpmLvxDN5bfQJD2oQgu8CIbfHXy32tuspsFlWyQNsPO85jw8lr2HDyGoNFLcJgQU61DvPF6sn9cfRKutNgodOo8NQtzRCfnGULFiWlwhdrzN2wBt1sYzXUMCFSSkZL6TKaSwm2wNFMugIPyYDW0iW0xiWHexmEGpdEA7vQcU6E4rSIQIIIYtcKgKEfbQYAHJg+GAH1dNh9zvkH4fif9uG/9xQtR302JRszVhzF9BFt7NYvsYYKZ0p2g5jNAoP/twlnrmXbHb/3qx0Ozz2VlIX2M9aiaVA9nE3Jdjhf3LCPN5d5viLWn0h2enzpXsd/Z1Y7zl7HjrOWv7+1xXbALW7DyWTc2ir4pstX213LzMeQ/23CnR3CMXOU44aUZXV7lScjV5kbhSkdgwWVqfj4DGdcHWRvghrnRRjOizCsLRY4VDAjUkpGlJSExlIiGkvJtq+NpGToJQOaSoloCsetzXOFDmdFGOJFBE6bIxAvIhAvwnFBhJa7P4oSXcvKR0A9HbadKf037OIDJQFg/vbzaBniA3UFf+t8btEBu++bvvq3y+UsL1QARd01VeFml65fuveS4oPFhevZeHrBfjx1S1OM6hjh9Jr528/hRo4BP+y44DRYFGc0mfHo/D1oE+6LV25v7fQaIQR+j0tAmzA/l3++UM1Q937qkktaBHujX4sgbDmdgsXje+L5RQeQnJmPoTEh5T/ZBWaocEGE4oIIBWC/uZMEM0JxA41VSWgsWf8koql0FU2lq/CUChAjXUAMLth1rRiEGhdECOJFBM6IMFwWDXBVBCJBBOGKCEQ2PKEUzuLAtcx8J0ctVhy84nDsZGIGNCq2/FSUHGuHVMal1BxMWLAP4/uXHg5K89ryIzh2NQOTFse59Nwley/hr0NXkWcw2bWCzfjzKLacTsGW0ykOwUIIAUmSsP5EMqb8Ylmb5OkBzVwqL9UMDBZUJkmS8NPjPWzfx065BaeTM9GlcfXtqCmgwlUE4qo5EDvRxu6ctWuluXTF1rXSTEpAc+kKvKU8y3E4fogClt1fE0QgrhQGDUvoCESiCIQkCXggH54ogCfy4SkVFH0vWb56FB6/Jvyx19wSe8ytcAPu27HXFdkFRfOCM/OMFVrToaQfdlxwZ5EUz5UW/gvXs/H670fw9IBm6N0sqOoK5cSryw/j6BVLOMg3mBET4YuYcD9sOnUNkQGeaNrA2+nz8o0mhzEtFfVSKa1BC3ZedHp834UbeOqnvZg+og0uXC+aRcLFnmsnBgtyiZ+XFl2j6tu+D/eTd4nv4l0r/6BLsXU3BMKQiuaqBDSXEtBUuoow6ToipOsIl1LgJ+XAt/CPszEdrhqPlQCA0+YI7DG3wm5zNPaKVrgsglDdqyqNnrvdbnAmVQ2jqeKfes8vOoCDl9Ox5XRKqWs5ZOQZcDQhAz2a1IdKJeG7reew+kgi5j3SFb4ejpsCnk/JxsXUHPRv2aDU15269CC2nC4KB9ZpvCue7YNx31lm2JRWnhkrjjo9Pvvv4ziVlIl547pBgmX9kMpYsPMCNp68hs/GdMLoudsBAJMWx+E/Q1rarim+jYS1RYOKGE1maMrprpYDgwXdlAkDmuGTKpjfH+rrgcSMosWMtGoJBhd+kAOSrZVjC9o7nK2HXIRJ1xFu+5OCcCkV4UhBiHQDZqiQCx1yoUeesHzNhQ65Qoc822M98qFFlJSEbqoTaKmytJi0UCVgDNYDAK6I+thjjsYecyvsMbfCKdGwWgaa/nPc+YBDcp/VRx3H+5Sm+L/l0tz1+TacuZaNd+9qhzE9GuGtv44BsCxH/vKwaIfrBxROjV32dG+nLYi5BSYs3XfZ4ThgWRK9PIt2Ow/cXxUulLYtPgUz/zxqN2B306lr+KmCLV+v/26ZrbN4t30rRvHwYH0tAGjyyt+YdVdb3N42DPXrVW4tnpri+NUMLNl7Cc/d1qLSdbmelY8BH2zE4JgQfHhvR/cW8CYxWNBN8dJpHEJARfz5bF+M/Gxrqed3vjoQUdNW3mzxSpUNT8SLhogXDW/6Xv1bNsBrp64hABnoqjqFbqqT6K46gRjpPMKlVIxSb8coteU3slyhQ7LwxzX4I0X44ZrwwzXhjxQ4Ps5H7f7hSa6xfkCvOJhgN7Wy5FodJR2+nIa2Eb549bcjuDW6AUa0DwcAmMvoR/h8Y9EvA+k5BszbehZ3dYpAQz8dDGbg8R/3OX3ezrNFA4K3nUlxmAVkbQVxRVaJKcWfbyj9F5XXlh/Bkr2X8cfEPqVeUxvc/vEWAMDVtDx8+VCXSt1jyd7LyMw34rf9CQwWpDxrpvRHh5lr8UD3SCRn5CM6zAefb3C+OqJVu4Z+eKp/U7vfSMpS/GdkmzBfHLvqvsWKSrufj16DzPzSV3G06t0sEJtPXcMN+CLW3BWx5q4AAE/koZMqHt2kk+imOoHOqnh4SfmWGS9wPgWyuAzhVTj2oz6uiCDbeJCrIhAJCESSqF8nZ70oXck8YO0OMJkFdp9LRfuGfvDS2a/psmDnRSzbfxnL9l/GiPbhyMwzOB2ga1V8tk2HtyxrgXy6Ph79mgdiS7wGgOOMoovXc3B/sSnIX22q2P/d8pRsicwpNl7ImYOX0tzyujVBaT/HzGaBk0mZaBXiU+H1QQ5fTsebK45g6pAW7ixipfCnEt00P0+tQz+tr4cWs1cV7Y56a6sG2HDSforj5EEtKxwsbmnZAOsK1yPw0KogSe4Z2NUi2Bv9WzZw+h+8YyN/tI3ww55zqfjp8R4Y8tEmp9MfhQCevbU5Pivxm1YuPLDd3Bbb0RYwAYEeQBvNVTzfLwTfrtmLBlIaGkhpCEI6GkjpaFkvB+rcFDRAGvSS0TYGJLqUMSBmIeEa/AoHn9ZHhqiHfGiL/ggd8qBFPnSF3xedM0MFNczQwAQVzNDADDVMlmOSCSoIaGCCGiaYoUKG8EI6vJEm6iEN3kgX9ZCBenV+pdRf9lyEh1aNpkHeaNfQ+b44RpPZtiAXAPy67zI6NPTDzD+P4ekBzdCnuf1gzl3nUu1WCz2SkAGzWaDNG6uRbzQjJtzXbuGwGX8eQ9Ogenb3aDej/IXDnNlSxiJg/T/YUKl7lufjdaer5L4VkZSRh0/Xn8ZDPaPQKtT5EvZVqbQhI3NWn8DXm8/ini4NMapjBHo1Cyx3KvjYeTuRkWfEA/P24ONeVVBYFzBYUJV46pZmOJSQjpWHLCsrzn2wC6KnrwYAzBhpmdnhWeK3rhMzB+PfH61B33aWKWb/vHALBn1o2XNk9uh26D5rHQBLH6yEiq+h8Z8hLRHi6+F03YKAEvumnJ8zHPd/vQM7z6ZibI9GGNY2zHZu039uhSRZ+nqLG9CqAVqH+eL+7pHo+57lh2/TBvXwYI/GCKintU2dWzl5AHZt+gcdew/D6lWOH8h/P9wPd3yyBYCAL3IQLN1AmJRaNAYE9uNB9JIBIUhDiJSGThX8u3C3dOGFNOGNdNSzfU0VPrgBH1wXvrghfHAdlq/W40pqZXl5WdHKpqUNgpz+h/3Kn/8p3OYdsKwk6ux5Iz8tWhH0cEI6vt5yFvlGM4Ci1VGLK74miJnbwFfY84sOYNe5VCzafQln3r2jSl7janouvtp0Fg/3alzqDJySrBv+Ld13GUv3Xcard0RjfP+yp96WtvGeHJTzP5xqnOLz1z20ajzetwm2nk7Bfd0cl+b10KqgVkl4MtqMOwZbmvKaB3tj8fieMJkFgn2KZp9EBngi7lJahZosFj7Zwza9z1mwcLZM8I+P9cClGzloVuKHgLMmyQ3/GYAmhb8tNgzwsh2XADzWtwkMJrMtWPh6lP7fbfXkfogO9cXBN4fgjo+3ICFNQoaoV8YYEIFAZNhmuoRJ1+GFPHhIBdDDUPRHMsAD1mMF0EuW4xqYYYAaZqhghMrSNiFUMEEFI9QwFR4zFrZs+CEbfpLljz+y4C1ZxtT4STnwk1zbZCpDeCJV+OIGfJAj9DBCjQJoYIQaRqhhgAZGYX1c+D00MEIFMyQIqGAWEsyQYIYKAlJhu4tkO2+AGteEP5JEABJFfSTDv1qXhBdCYOafx9A6zKfUQZBlScmyX4NkTrHWv/JUZrGy2i41uwBqlQQ/T8fZM2U5VmwJe3cRQuDolQykZhegX4sgTFiwHwcvpeGPuASM6hiB3FK6erLzjXhp2SEMbxfmcO63/Qno1TQIvx24jMkDW8LPS2vX2lHTwiSDBVWZuzs3xB9xV9AyxPIBPX1Em1Kvnf9od6fHezYNtD1e+GQPLNlzCW+MjEG3JvXt9oDw1mscBoEBcDpavl2EHw4npOOB7pEY1TEcQgBpOQXo0tgyjVanUTmEiuLeG90OLy87jFYhPrZQURqtWoWDbwyBgLBtZuVMqK8lOPl5ajF5UAtbCLIOjH1xcEss238Z5wvn+B+eMRTtZqzFdeGHI6JpmWWoCloY4Yts+EtZttDhjywESFnwlzIRiEwESJmoL2WiPjIsj5EJtSTgK+XCV8pFFKpv5opZSEiBHxJFAJJEfSQWBo5EUR9JCECW8EQeLF1HuUJf+NjShVTR6cJJGXn4Zc8l3N89EqcSszB/+/kKl6+gsDWCXJdnMKHz27EAgLPv3uHSniUVmb0qhMD17AIEeesrdM93Vh7Ht1vPAQDm3N3ONibkRo7B4d9E8Zf/avNZrDx01dbKW5J1sHtmntFuSX4AuO3/Ntp9f921sfRux2BBVeaWlg3wzwv97X6TL+n90e1xLSsfPZsGwmAoe/R772ZBttaHsT0ao224H1769RCah3jj4Z6N7fa2eKR3FF65Ixp6TdFvqQFeWtzIMWDx+J5IycpHo/pelm4VCXikT5MK1+verpFo2sAb0RXsk/XzsvwWVVb9ik+xa+BT9ANs56sDkWcwwUOrRoHJjE/XxyMm3Bc+TtY1qE4GaHAdfrguCscVVOAXJglm+CIH9aVMBCATgVIGPAvbKjSSCVqYoIEJWhgLH1uO62CEpvAcYFn+XQUBta19QliOScLangE9DAiRbiBEuoFgpEErmRCMNARLaQDOuVTXPKG1TDuGDnlCC2MpPzbT/wsMA5C2CWjmocVanQEmqFBgG/NSbPwLdLbvsXYHvt2WgOfVKuilAnghHx4ogJeUb1mcrXAhNsvxfNvxAmiQLTyRDQ9kwRPZwsPyuPBYNjyRJSxfcwqnRhdAYymP0KKg8Pv8YuUrgNblQCW3q+lFn6JGs4CunGBhMJmxLUnCpOkVG4cye5VlvMP/3dMBo7uUPYsst8BkCxUAMO03x00AS/NJBcearIi7gif6NcHH/xRdf/66favhWwc0eOjuCr+02zFYUJVqHlz2h++93SIrfe8Okf5YM6U/AOBIQtFGaL9O6IWOkf4OC8fsfHUgjCaBenoN6ukr/09fkiR0K7ZIWGnXlEWnVqHAVPRbar1i401uadkAz93WHG3CLKt4emgt5567rQXaRvihZ5NA1EYCKqTDG+nCG+cQ5vpGM5UkwYwgZCBESkWIdAOh0g2ESKkIxQ2ESqkIltJQr7AbyQOWP1qpqLnaQzLAAwbrzSqmAAir6LjW7WvxtASgElmxgVQ1W7kXCDUy4YUM4YVMeCFTeCGj8KvdcXhCDTN0MEAHo+WrZCx6XOKYGZIl0AiNXdApEBpb0LEGMbOQoJOM0NruZYIOBmglI7BmO2AyAKZ8BGXn4P+0V5AlPKHauA/wqg94+gMe/oVf/WyPt17Iw4Pf7QaghgQz6iEP3shFPSkPuLwXyM+EITcD6WnXEaQ14GzCVXjuu4jn1BKu/rUCyGkJSCpAUhd+LfyjUuNKRj4+2XAOo1SWdW9y4IFcYX2sR67QF66Ho7eNM6rMgl8FJjOGfbTFje+2+zFYkCIEFFtkpmspH/p6jRo3kScqVo7CVpH+LUpfDREAjr89DC8vO4Rwf0880a+JXQiSJAkvDmnl8BydRoWhMaFuL7PSCahwDf64JvxxpIJhRg2TLWR4SgXQoyh0aODYbSFJzm+shqXFpeQYl6IxMEVjYnQwIg865EKHHOFhW5QtR+iRZ/tw0iEXHsiDFlqYUA+58JbyUK/ww9EbuZYPy2LHfJALD+RDJxmLvZbl9fRSsccogLqwHjrJhEBkIlDKrPTfe5UqtmmuD4DR1ly+NbbMp/UQauzXe0IHo22ckM08yxctAOs8naYAplgDnwCwrvR7hwOYU8FwaBBq5EKP/Ewt8j7wgd7DC3/pCpAPLfKEztaKlFfYspWXogM0KNZSZ4YaAmrJ+thsa7mzPkZeT0Arzy8hDBakCBH+nnj/3+2dLn1cnVY+3w/rTyRjdOeym0zVKsmhn5RqDhPUyIanZaO6ioQROcfOufG11TDBE/nwQS58pBz4Ihs+Ui58ULgEfuFXH+TAR8qBN3ILu3usLQ/FWiFs3S6WrwZooIK5RCtGUajRS/atHGqYba0aBmhs9zZAg0f7twLUOiTnCsRfL8DGUynwkXLR3MeA25t5AnlpQG4akJcO5KXBnJsGldkArWRCfWTZ1dkoVMiCp+WPKP7VA9nCEwaoC4cIW7rdrB/m3loV8gxGqGFG61BvnEpMhxbGEt1YlsceyIcX8qGRLKFUK5mgRWH3RXY6kA20dfPMbYOx6nYGLg+DBSnGvV0r363iLuH+nniwZ2O5i0FUKSaokQUvZMELV0Xhb7s1a8IBACCyYVcMahOC7iVX570BPO/XAi/827LfyLb4FDy/6ACu51g+6P2QDV8pB/nQIlt4IhOelR9PUnzIVIUm/ghoYYIn8uBl29jQ0mJlnc1lm8FV4nsPqQCAZRdosyg+C0pl97joexVm6io2tbUqMFgQUZ0zskM4/ixjZUqq2Z74cS+2TbvN6blP1p3G9vgUjOoYjul/WDdSk5ALD+TCA4lCrjFKEgzQwABvZMDbPrBVQXibqSt7xlpVqtvL5hHVYgFe8nb71GYTby17sSFntk+7DY3qe1ZBaagyziRnlXpu74UbxUIFVTcGC6Jaavbdjru2KsGOV27D8HZhaF/KEtlVaevLtzocW/5Mb5yfMxzh/p5Y+Wzvai8TOfdwJTY8q0uEO/Y8qCQGCyIC4HwxMVfo1GX/OCnv/q/eEY3t025DmJ8nPh/bGR0a+lfodR/pHVXquXA/D7wwuGWF7tO3eRAaBnjhsRJrmvgWW83ROvWXqKaTczFOBguiOub0rNsRHeqDIW1C7I5/PqZzpe63dEIvnJ8zvNzFg74YW/r9H+zZCOP7N0O4f1FXwyN9oir0+rdFB9t937Np0XTjNVP6Y1zvKLQKKX09lalDW+H8nOFY8EQPAMD0Ea3xy/ietvMVXXGRqCYxypgsGCyIaqmKjrEo/sH43SNdoVWrsGpSP3z9cFf0aV40kC3Uz8PZ08vVtbAlYvqI1hjcOhht/J0vTx3i64EVz/ZxaGEIrKfDO/9q53B9swbeODpzKKICi1Zu9ffS4p4SASY6zD403NkhAgAQFegFHw8t/Dy1toXUrKQyZgFIkoQeTQOxdkp//PlsX5f3nyCqCeTcP4SzQohqqe5N6mPCLc0Q7u+BN8oYqNYx0h//HLfsy3FbtKWVwrri3zcPd8Un6+Jxe1vnC2/d3jYUAfV0WLjrIga1DsYd7cKwaPdFPNmvKcb/tA9TBrW03ctLp8EXYzri77//Rqc+A9Cwvjf2XriBe78qWs2ofUN/tG/oj2NXM7D7XCoAYO/rg0otez29Bp+N6YzHf9iDl4ZGY1THcBSYzFi677LtmuIb1AHA/d0i0TjQC23D7cdo+HponO4AWVpfdMsyWjmIajo5WywYLIhqKUmSMO32aABwCBYtgr1xOjkLnRr5o6y5bF46je0exX35YGdsi7+OyYNawNtDg8GtQ9CjaX146TS4u3Dxr9K2CQeAMD8PqFQSSht28caINhjx6VYMah1S7rLGbSP8sPOVgbbrNGoV9r4+CN9sPov7nCwJr1JJ6NM8yOH41mm3of0Mx/0hQnwr11JDVJO5c8dWVzFYECnQj493x+LdlzC2ZyO8sqziGyEterIn0nMNGNY2FMPaFm3ffGuJcQwV1TEyAL2aBqJxoP1GdG0j/HDwjSHw9azYj6CS4SPIW49X7mjtUlm8dfav9c3DXbHz7HVbUCL3CPLWO2z7XpruTerbWq4A4MN7O+CFJQerqmh1isks3465HGNBpAC7Xx2I/93XAd2iArB4fE+E+XliyuCWCPbxKGy1qJhezQIxrJRukcpQqyQsGt8Tc0Y7To3189JWahMmZ/oWtlB0iPQv9ZqSLzW4TQimj2gDtQvbbNdUHapwam6TIOcLLWlUkm2jvOJC/YrG9IT6emD3awNLvfe4XlE3XT5yziTjrBC2WBApQLCvB+7q1BB3dXL87fvJ/k3hqdPglpaO3QNK8ekDnfDbgQSM6hheoeu9PWrHj76Xh0XjvdUnnJ7z99IiLceyrnR502BfHNwS/xd7qtTzHRr6IcTXA2oJWHU0yXZ8wi3NMO32aBy/moFvNp/FlMEt8fXms/hp5wUsGt8T3aLq44+4BPy2PwGN6nvhp50X8PrwNrj/6522e5QcA1Nc6xIDb/2LDUg+MH0wftl7CTdyCrB49yWk5xpKPp3KEFhsY8bqVjv+dxFRpek1ajzet0n5F9ZiAfV05dZRkiR88O/2yMwzIsK/Zq6g+eod0Xj3b0uQODRjCHw9tBjUOhiD/7fZds2Sp3rht/2X8cyA5uj/wQYAlp1vy/LcwBalBotfJ/RCl8YBkCQJBQUFuPWvVdiU2xArDydibI9GAIDWYb748L6OAIC3RsXg5duj4V24VfCojhEY1dEyE+e14a3hoVWjaYN6OHst22nr1+vDWyM1uwC9mwWhaQNveGrVyDVYtqnv27wBhsWEomtUAALq6TDhFssKqQ/2aIy3/jqG2GNJDvdTmr2vD0LXd/6p1HO3vHQr/j13O7oF5MjaEsdgQUR1xj3VvFHd1KGtcF+3SLyw5CA2n7rmcH5kh3B8+kAnXLyeg4YBnth57rrtnHWn3uLTXfe8NggNfPTo3sSyVsdrd7TG8gMJeKp/M2w5neK0DNb1SZY81ctuhk63qAAE1tOja1TRuh+SJEGvBv53Tzv8370dnbaESJJkCxUlWa9f8lQvbDp5DcPbW8bpDI0JwZrClpBAbx2e6NfU9pz3/90ezy06gKlDW0GnUeHLh7o43Deyvhdm3dW2TgSLyq6bEu7ngcj6XtgytT9WrVrl5lK5hsGCiMiNvPUaZOVbprU+M6AZJEnCj491R1SxnTinDm2FoTGhaB5s2YGyUeHg1u5R9dGhoR+aNSjamTLY1wP3d4uEVq1CAx/7D50n+zfFk/2b4khCutPXP/H2MNuHffcm9dGsQT2cuZYNAFg6ofTlySVJuqlVRoO89XYLpn0xtguavfo3AKDk7N6RHcJxa3RwqWHFyktXOz6uOjfyx/6LaaWebx7sjfjCfU7u7hSB3w4kVOp1Dr4xBB3eKprl9O0j3QA4DnSWQ+14p4iIaoghEWasTSi962Hx+J74ctMZPNGvaak/5Cfe2tzpcY1ahT+e7etw3Nng1+KKj02Ie2MwTidnwUundggHcx/sgqlLD2JyBZc5dxe1SkKvpoHYeyEVA6NDHM6XFyoqek1N8NVDXXEjpwC5BSb8X+wp5BlMdjNftMXmYI/sEI53726H6OmrAViCaEW0DvOFn5cWO165DQM+2IjRXRqitZOBtHKpHe8UEVENMbyRGXMeGYgJPx/E7vOWD4z3R7dHu4Z+MAuBmHA/fFbG8uiD2zh+sN6shgFeeH90e/h6aqBRq0r9kGkZ4uM0uFSHhU/2QIHJDL2mdu238t97OuBKWi4+LGPwa3EqqWhxtR8f6w4hBFpNX40Co2X656SBLTBhwT7b9R5aNVqGeONUUhZGtLcMPj74xhB8tfkMvth4xnbd+6Pb46VlhwAULeoW5ueJozOHQlPOPj3VrWaVhoioFvDx0OKtf8XA30uL14e3xr3dItE6zBcx4eVP+xwW477pvMXd2y3Sbu2RmkaSJFlDhX8Fl8Cf/2g3u+//3aUhnhnQDGqVBE0FBkSWnOUpSRJGd46wfV98QKsovPqv5/phxyu3oU24JRD6eWlxb7HxQH891xf3OlkMDkCNCxUAgwURUaVEh/pi/+uD7QYiluWX8T3xnyEtcVeniPIvpnKdm32H7bFPiW6SYTGhWPRkT7w0rBWCvPX48sHOiHtjCM7PGY7Tbw/B5LZG/Dahh9P7DmgVjPAS++Zo1CocmTEUh2cMtU3jDPLW4a1RMQCAcb0a267VO5mhM2VwS0SH+mDGyDZ2x61rWOk0KoT52c9UKt6L1iLEMubm7VEx8PPU4r/3dHBa9pqCXSFERJWkcmFKX4+mgejRNLD8C6lCJEnC9mm3IT45C+0i/NDp7VgAwJsj2+DRPpapx72aBeKZAY7jWZr4AO0i/LB4fE98tekM8o1mbD9z3bbQ2r+7NMQn6+PRsdiCa546S2vL5pduxfIDCRjcJgQhvh64p0skPHVqdGtSH0JYWrNKCvbxwOrJ/R2OlxyMW1y4vye8dGp46dTQFbZKPNQrCmN7NHbp350cGCyIiKhWCvf3RHjhmiTn5wyH0WR2qWugZ9NA9GwaiDyDCetPJKNfC0uweG5gC3RqFIAuUQEOz6mn1+DBnkUtFNbAYR0fURHfPNwVF1NzylwpVqtWYf/0wVBJkt0g4JoeKgAGCyIiUojKjjfw0KpxR7ui8SlatarS++NUREUH8N7MlF85cYwFERERuQ2DBREREbkNgwURERG5DYMFERHVCr0KZ9X0bFq/nCtJThy8SUREtcIXYzvjz0NXMNKFGRhU/RgsiIioVgiop8PDvaLkLgaVg10hRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNgwWRERE5DYMFkREROQ2DBZERETkNgwWRERE5DbVvrupEAIAkJGR4db7GgwG5OTkICMjA1qt1q33rgmUXj9A+XVUev0A5deR9av9lF7Hqqyf9XPb+jlemmoPFpmZmQCAyMjI6n5pIiIiukmZmZnw8/Mr9bwkyosebmY2m3HlyhX4+PhAkiS33TcjIwORkZG4dOkSfH193XbfmkLp9QOUX0el1w9Qfh1Zv9pP6XWsyvoJIZCZmYnw8HCoVKWPpKj2FguVSoWGDRtW2f19fX0V+Y/FSun1A5RfR6XXD1B+HVm/2k/pdayq+pXVUmHFwZtERETkNgwWRERE5DaKCRZ6vR5vvvkm9Hq93EWpEkqvH6D8Oiq9foDy68j61X5Kr2NNqF+1D94kIiIi5VJMiwURERHJj8GCiIiI3IbBgoiIiNyGwYKIiIjcRjHB4vPPP0dUVBQ8PDzQo0cP7N69W+4iOdi8eTNGjhyJ8PBwSJKE33//3e68EAJvvPEGwsLC4OnpiUGDBuH06dN216SmpmLs2LHw9fWFv78/Hn/8cWRlZdldc+jQIfTr1w8eHh6IjIzE+++/X9VVAwDMnj0b3bp1g4+PD4KDg/Gvf/0LJ0+etLsmLy8PEydORGBgILy9vTF69GgkJSXZXXPx4kUMHz4cXl5eCA4OxtSpU2E0Gu2u2bhxIzp37gy9Xo/mzZtj/vz5VV09AMDcuXPRvn172+IzvXr1wqpVq2zna3v9SpozZw4kScLkyZNtx2p7HWfMmAFJkuz+REdH287X9voBQEJCAh588EEEBgbC09MT7dq1w969e23na/PPmqioKIf3T5IkTJw4EUDtf/9MJhOmT5+OJk2awNPTE82aNcPbb79ttz9HjX//hAIsXrxY6HQ68d1334mjR4+KJ598Uvj7+4ukpCS5i2bn77//Fq+99pr47bffBACxfPlyu/Nz5swRfn5+4vfffxcHDx4Ud955p2jSpInIzc21XTNs2DDRoUMHsXPnTrFlyxbRvHlz8cADD9jOp6eni5CQEDF27Fhx5MgRsWjRIuHp6Sm++uqrKq/f0KFDxffffy+OHDki4uLixB133CEaNWoksrKybNdMmDBBREZGinXr1om9e/eKnj17it69e9vOG41G0bZtWzFo0CBx4MAB8ffff4ugoCDxyiuv2K45e/as8PLyEi+88II4duyY+PTTT4VarRarV6+u8jquWLFCrFy5Upw6dUqcPHlSvPrqq0Kr1YojR44oon7F7d69W0RFRYn27duLSZMm2Y7X9jq++eabIiYmRly9etX259q1a4qpX2pqqmjcuLF45JFHxK5du8TZs2fFmjVrRHx8vO2a2vyzJjk52e69i42NFQDEhg0bhBC1//2bNWuWCAwMFH/99Zc4d+6cWLp0qfD29hYff/yx7Zqa/v4pIlh0795dTJw40fa9yWQS4eHhYvbs2TKWqmwlg4XZbBahoaHigw8+sB1LS0sTer1eLFq0SAghxLFjxwQAsWfPHts1q1atEpIkiYSEBCGEEF988YUICAgQ+fn5tmtefvll0apVqyqukaPk5GQBQGzatEkIYamPVqsVS5cutV1z/PhxAUDs2LFDCGEJXyqVSiQmJtqumTt3rvD19bXV6aWXXhIxMTF2r3XfffeJoUOHVnWVnAoICBDz5s1TVP0yMzNFixYtRGxsrLjllltswUIJdXzzzTdFhw4dnJ5TQv1efvll0bdv31LPK+1nzaRJk0SzZs2E2WxWxPs3fPhw8dhjj9kdu/vuu8XYsWOFELXj/av1XSEFBQXYt28fBg0aZDumUqkwaNAg7NixQ8aSuebcuXNITEy0q4efnx969Ohhq8eOHTvg7++Prl272q4ZNGgQVCoVdu3aZbumf//+0Ol0tmuGDh2KkydP4saNG9VUG4v09HQAQP369QEA+/btg8FgsKtjdHQ0GjVqZFfHdu3aISQkxHbN0KFDkZGRgaNHj9quKX4P6zXV/X6bTCYsXrwY2dnZ6NWrl6LqN3HiRAwfPtyhHEqp4+nTpxEeHo6mTZti7NixuHjxIgBl1G/FihXo2rUr7rnnHgQHB6NTp0745ptvbOeV9LOmoKAACxYswGOPPQZJkhTx/vXu3Rvr1q3DqVOnAAAHDx7E1q1bcfvttwOoHe9frQ8WKSkpMJlMdv9IACAkJASJiYkylcp11rKWVY/ExEQEBwfbnddoNKhfv77dNc7uUfw1qoPZbMbkyZPRp08ftG3b1vb6Op0O/v7+DuVzpfylXZORkYHc3NyqqI6dw4cPw9vbG3q9HhMmTMDy5cvRpk0bxdRv8eLF2L9/P2bPnu1wTgl17NGjB+bPn4/Vq1dj7ty5OHfuHPr164fMzExF1O/s2bOYO3cuWrRogTVr1uDpp5/G888/jx9++MGujEr4WfP7778jLS0NjzzyiO11a/v7N23aNNx///2Ijo6GVqtFp06dMHnyZIwdO9aujDX5/av23U2pbpg4cSKOHDmCrVu3yl0Ut2vVqhXi4uKQnp6OX3/9FePGjcOmTZvkLpZbXLp0CZMmTUJsbCw8PDzkLk6VsP7mBwDt27dHjx490LhxYyxZsgSenp4ylsw9zGYzunbtinfffRcA0KlTJxw5cgRffvklxo0bJ3Pp3Ovbb7/F7bffjvDwcLmL4jZLlizBzz//jIULFyImJgZxcXGYPHkywsPDa837V+tbLIKCgqBWqx1G/SYlJSE0NFSmUrnOWtay6hEaGork5GS780ajEampqXbXOLtH8deoas8++yz++usvbNiwAQ0bNrQdDw0NRUFBAdLS0hzK50r5S7vG19e3Wj4YdDodmjdvji5dumD27Nno0KEDPv74Y0XUb9++fUhOTkbnzp2h0Wig0WiwadMmfPLJJ9BoNAgJCan1dSzJ398fLVu2RHx8vCLew7CwMLRp08buWOvWrW3dPUr5WXPhwgX8888/eOKJJ2zHlPD+TZ061dZq0a5dOzz00EOYMmWKrQWxNrx/tT5Y6HQ6dOnSBevWrbMdM5vNWLduHXr16iVjyVzTpEkThIaG2tUjIyMDu3btstWjV69eSEtLw759+2zXrF+/HmazGT169LBds3nzZhgMBts1sbGxaNWqFQICAqq0DkIIPPvss1i+fDnWr1+PJk2a2J3v0qULtFqtXR1PnjyJixcv2tXx8OHDdv8pYmNj4evra/th2atXL7t7WK+R6/02m83Iz89XRP0GDhyIw4cPIy4uzvana9euGDt2rO1xba9jSVlZWThz5gzCwsIU8R726dPHYZr3qVOn0LhxYwDK+FkDAN9//z2Cg4MxfPhw2zElvH85OTlQqew/mtVqNcxmM4Ba8v7d9PDPGmDx4sVCr9eL+fPni2PHjonx48cLf39/u1G/NUFmZqY4cOCAOHDggAAgPvzwQ3HgwAFx4cIFIYRlCpG/v7/4448/xKFDh8SoUaOcTiHq1KmT2LVrl9i6dato0aKF3RSitLQ0ERISIh566CFx5MgRsXjxYuHl5VUt002ffvpp4efnJzZu3Gg3HSwnJ8d2zYQJE0SjRo3E+vXrxd69e0WvXr1Er169bOetU8GGDBki4uLixOrVq0WDBg2cTgWbOnWqOH78uPj888+rbSrYtGnTxKZNm8S5c+fEoUOHxLRp04QkSWLt2rWKqJ8zxWeFCFH76/jiiy+KjRs3inPnzolt27aJQYMGiaCgIJGcnKyI+u3evVtoNBoxa9Yscfr0afHzzz8LLy8vsWDBAts1tf1njclkEo0aNRIvv/yyw7na/v6NGzdORERE2Kab/vbbbyIoKEi89NJLtmtq+vuniGAhhBCffvqpaNSokdDpdKJ79+5i586dchfJwYYNGwQAhz/jxo0TQlimEU2fPl2EhIQIvV4vBg4cKE6ePGl3j+vXr4sHHnhAeHt7C19fX/Hoo4+KzMxMu2sOHjwo+vbtK/R6vYiIiBBz5syplvo5qxsA8f3339uuyc3NFc8884wICAgQXl5e4q677hJXr161u8/58+fF7bffLjw9PUVQUJB48cUXhcFgsLtmw4YNomPHjkKn04mmTZvavUZVeuyxx0Tjxo2FTqcTDRo0EAMHDrSFCiFqf/2cKRksansd77vvPhEWFiZ0Op2IiIgQ9913n90aD7W9fkII8eeff4q2bdsKvV4voqOjxddff213vrb/rFmzZo0A4FBmIWr/+5eRkSEmTZokGjVqJDw8PETTpk3Fa6+9ZjcttKa/f9w2nYiIiNym1o+xICIiopqDwYKIiIjchsGCiIiI3IbBgoiIiNyGwYKIiIjchsGCiIiI3IbBgoiIiNyGwYKIiIjchsGCiIiI3IbBgoiIiNyGwYKIiIjchsGCiIiI3Ob/AQduMdN7QtKEAAAAAElFTkSuQmCC"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 推理\n",
    "\n",
    "- 翻译项目的评估指标一般是BLEU4，感兴趣的同学自行了解并实现\n",
    "- 接下来进行翻译推理，并作出注意力的热度图"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "# load checkpoints\n",
    "model = Sequence2Sequence(len(src_word2idx), len(trg_word2idx))\n",
    "model.load_state_dict(torch.load(f\"checkpoints/{exp_name}/best.ckpt\", map_location=\"cpu\"))\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "        \n",
    "    def draw_attention_map(self, scores, src_words_list, trg_words_list):\n",
    "        \"\"\"绘制注意力热力图\n",
    "        \n",
    "        Args:\n",
    "            - scores (numpy.ndarray): shape = [source sequence length, target sequence length]\n",
    "        \"\"\"\n",
    "        plt.matshow(scores.T, cmap='viridis')\n",
    "        plt.xticks(range(scores.shape[0]), src_words_list)\n",
    "        plt.yticks(range(scores.shape[1]), trg_words_list)\n",
    "        plt.show()\n",
    "        \n",
    "    def __call__(self, sentence):\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_input, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()], \n",
    "            padding_first=True, \n",
    "            add_bos=True, \n",
    "            add_eos=True, \n",
    "            return_mask=True,\n",
    "            )\n",
    "        encoder_input = torch.Tensor(encoder_input).to(dtype=torch.int64)\n",
    "\n",
    "        preds, scores = model.infer(encoder_input=encoder_input, attn_mask=attn_mask)\n",
    "        \n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=False)[0]\n",
    "        \n",
    "        src_decoded = self.src_tokenizer.decode(\n",
    "            encoder_input.tolist(), \n",
    "            split=True, \n",
    "            remove_bos=False, \n",
    "            remove_eos=False\n",
    "            )[0]\n",
    "        \n",
    "        self.draw_attention_map(\n",
    "            scores.squeeze(0).numpy(), \n",
    "            src_decoded,\n",
    "            trg_sentence\n",
    "            )\n",
    "        return \" \".join(trg_sentence[:-1])\n",
    "        \n",
    "    \n",
    "translator = Translator(model.cpu(), src_tokenizer, trg_tokenizer)\n",
    "translator(u'hace mucho frio aqui .')\n",
    "translator(u'esta es mi vida.')    "
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-15T07:26:00.225605Z",
     "iopub.execute_input": "2023-12-15T07:26:00.226304Z",
     "iopub.status.idle": "2023-12-15T07:26:01.386679Z",
     "shell.execute_reply.started": "2023-12-15T07:26:00.226256Z",
     "shell.execute_reply": "2023-12-15T07:26:01.384718Z"
    },
    "trusted": true
   },
   "execution_count": 34,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 480x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAGkCAYAAACsHFttAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmJElEQVR4nO3de1yUdd7/8fdwGk6Cggc0EUrDlTytaWquiYKZre52YlczD63po7rzVju52F3iISlXs26910N1l92lW5Zmm1YGHc1EMyEDblttSU0sxRbEwyjM9fujn3M7ooY1w8V8fT0fj+uxzcw1F5+vwLy4ZgbWYVmWJQAADBRk9wAAAPgLkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkTtNWlqaHA6HHA6HCgoKbJmhtLTUM0PXrl19euy0tDRNmjTJp8c0WXJysp588km7x/CwLEvjx49XXFzceb9GHQ6HXn/99XqdraHJzs72+fePyRrCY5+/ZiByZxg3bpzKysrUsWNHr+A4HA6FhYWpXbt2mjVrls78a2hFRUX6wx/+oGbNmsnpdColJUWPPPKIjh496rVfYWGhfve736l58+YKDw9XcnKy/vjHP+r777+XJCUmJqqsrEz33Xdfva0ZgeHtt9/W888/rzfffNPzNXo2ZWVlGjx4cD1P17Dcf//9ysvLs3uMgHK+x77Tt02bNnnuc+zYMU2bNk0pKSlyOp1q2rSpMjMzVVRU5HXso0ePKisrS23btlV4eLiaNWumfv36ac2aNZ59Vq1apc2bN/t8XSE+P2KAi4yMVEJCgtd1ubm5uuKKK+RyubRhwwbdcccdatmypcaOHStJ2rRpkzIyMpSRkaG1a9eqRYsW2rx5s+677z7l5eXp/fffV1hYmA4cOKD09HQNGTJE77zzjho3bqzS0lK98cYbOnLkiCQpODhYCQkJio6Orve1o2HbtWuXWrZsqauvvvqst584cUJhYWG1vn4vRtHR0XwPXaDzPfadLj4+XpLkcrmUkZGh3bt3a968eerZs6e+++475eTkqGfPnsrNzVWvXr0kSXfeeafy8/O1YMECpaamqry8XBs3blR5ebnnuHFxcaqsrPT9wix49OvXz5o4caLn8j//+U9LkrVt2zav/dLT0627777bsizLcrvdVmpqqtW9e3erpqbGa7+CggLL4XBYjz32mGVZlrV69WorJCTEOnny5E/OMm3aNKtLly6/aD1n6tevnzVhwgTrgQcesJo0aWK1aNHCmjZtmuf2efPmWR07drQiIyOt1q1bW3fddZd1+PBhr2Ns2LDB6tevnxUREWE1btzYuvbaa61Dhw5ZlmVZNTU11uzZs63k5GQrPDzc6ty5s7Vy5UqfzX7PPfdYEydOtBo3bmw1b97cWrp0qVVVVWWNGTPGio6Ottq2bWutW7fOsizLeu6556zY2FivY6xevdo680v+jTfesLp37245nU4rPj7euuGGGzy3JSUlWY8++qh1++23W9HR0VZiYqK1ZMkSr/t/8cUXVv/+/a3w8HArLi7OGjduXK1/M18YPXq0JcmzJSUlWf369bP+7d/+zZo4caIVHx9vpaWlWZZlWZKs1atX1/uMdfXWW29Zffr0sWJjY624uDjrt7/9rbVz507P7fn5+VbXrl0tp9NpXXnlldaqVau8vg/r8rn1x/ePyer62He6xx57zHI4HFZBQYHX9TU1NVb37t2t1NRUy+12W5ZlWbGxsdbzzz//k3PU5eNeKJ6uvECfffaZtm7dqp49e0qSCgoKVFxcrHvvvVdBQd7/nF26dFFGRoZWrFghSUpISFB1dbVWr15d6+nO+rJs2TJFRUUpPz9fc+bM0YwZM/Tuu+9KkoKCgvSf//mfKioq0rJly/Tee+/pwQcf9Ny3oKBA6enpSk1N1aeffqoNGzZo6NChqqmpkSTl5OTohRde0OLFi1VUVKTJkyfrtttu04cffuiz2Zs2barNmzdrwoQJuuuuu5SZmamrr75an3/+ua699lqNHDmy1lPE57J27VrdeOONuv7667Vt2zbl5eXpqquu8tpn3rx56t69u7Zt26a7775bd911l3bs2CFJOnLkiAYNGqQmTZpoy5YtWrlypXJzc3XPPff4ZL2ne+qppzRjxgy1bt1aZWVl2rJli6Qf/03CwsL0ySefaPHixbXuV58z1tWRI0d077336rPPPlNeXp6CgoJ04403yu12q6qqSkOGDFFqaqq2bt2q7Oxs3X///bbNinNbvny5Bg4cqC5dunhdHxQUpMmTJ6u4uFiFhYWSfnzsW7dunQ4fPlz/g/oslwY4108zERERVlRUlBUaGmpJssaPH+/Z529/+9t5f/L493//dysiIsJzeerUqVZISIgVFxdnXXfdddacOXOs/fv317qfv87kfvOb33hd16NHD2vKlCln3X/lypVWfHy85/Lw4cOtPn36nHXf48ePW5GRkdbGjRu9rh87dqw1fPjwXzh57dmrq6utqKgoa+TIkZ7rysrKLEnWp59+Wqef9nv37m2NGDHinB8zKSnJuu222zyX3W631bx5c2vRokWWZVnW0qVLrSZNmlhVVVWefdauXWsFBQWd9XP6S82fP99KSkryXO7Xr5/161//utZ+Ou1Mrr5n/DkOHDhgSbK2b99uLVmyxIqPj7eOHTvmuX3RokWcyfnZTz32nb6dEh4e7nWf033++eeWJOvll1+2LMuyPvzwQ6t169ZWaGio1b17d2vSpEnWhg0bat2PMzmbvPzyyyooKFBhYaFeeeUVrVmzRn/+85+99rHqeGb26KOPav/+/Vq8eLGuuOIKLV68WL/61a+0fft2f4xeS+fOnb0ut2zZ0vOml9zcXKWnp+uSSy5Ro0aNNHLkSJWXl3vOjE6dyZ3Nzp07dfToUQ0cONDzekh0dLReeOEF7dq1y+ezBwcHKz4+Xp06dfJc16JFC0nyrOennG89Z/uYDodDCQkJnuOXlJSoS5cuioqK8uzTp08fud1uz9mev1155ZXnvb0hzHimf/zjHxo+fLguu+wyxcTEKDk5WZK0e/dulZSUqHPnzgoPD/fs37t3b1vmxP899p2+na6uj3vXXHONvv76a+Xl5emWW25RUVGR+vbtq5kzZ/pham9Erg4SExPVrl07dejQQZmZmZo0aZLmzZun48ePKyUlRdKPDyZnU1JS4tnnlPj4eGVmZmru3LkqKSlRq1atNHfuXL+vQ5JCQ0O9LjscDrndbpWWlmrIkCHq3LmzXnvtNW3dulX/9V//JenHNzRIUkRExDmPW1VVJenHpwBP/4YoLi7Wq6++6rfZT7/O4XBIktxut4KCgmp9A548edLr8vnWc76P6Xa7L2hufzo9XoFi6NChOnTokJ5++mnl5+crPz9f0v99nf2Uunxu4RunHvtO305JSUk57+PeqX1OCQ0NVd++fTVlyhStX79eM2bM0MyZM+v8ef+5iNzPEBwcrOrqap04cUJdu3bVr371K82fP7/Wg19hYaFyc3M1fPjwcx4rLCxMbdu29by70i5bt26V2+3WvHnz1KtXL6WkpGjfvn1e+3Tu3Pmcb8tOTU2V0+nU7t27a31TJCYm1scSvDRr1kyHDx/2+nc986fQ862nLjp06KDCwkKvj/HJJ58oKChI7du3/9nH9aWGNmN5ebl27Nih//iP/1B6ero6dOigH374wWveL774QsePH/dcd/pb1qW6fW7hf8OGDVNubq7ndbdT3G635s+fr9TU1Fqv150uNTVV1dXVXp9rfyBydVBeXq79+/dr7969euutt/TUU0+pf//+iomJkcPh0LPPPqvi4mLdfPPN2rx5s3bv3q2VK1dq6NCh6t27t+cXsN98803ddtttevPNN/XVV19px44dmjt3rtatW6ff//73tq6xXbt2OnnypBYsWKCvv/5a//M//1PrjQxZWVnasmWL7r77bn3xxRf63//9Xy1atEgHDx5Uo0aNdP/992vy5MlatmyZdu3apc8//1wLFizQsmXL6n09PXv2VGRkpKZOnapdu3Zp+fLlev755732mTZtmlasWKFp06appKRE27dv1+OPP17njzFixAiFh4dr9OjR+vLLL/X+++9rwoQJGjlypOepU7s1tBmbNGmi+Ph4LV26VDt37tR7772ne++913P7rbfeKofDoXHjxqm4uFjr1q2r9SxHXT63gWDhwoU/+XS53U499p2+nYrS5MmTddVVV2no0KFauXKldu/erS1btujmm29WSUmJnn32Wc+zK2lpaVqyZIm2bt2q0tJSrVu3TlOnTvU8jvoTkauDjIwMtWzZUsnJyRo/fryuv/56vfzyy57br776am3atEnBwcEaPHiw2rVrp6ysLI0ePVrvvvuunE6npB9/comMjNR9992nrl27qlevXnrllVf0zDPPaOTIkXYtT9KP7wR94okn9Pjjj6tjx4566aWXlJOT47VPSkqK1q9fr8LCQl111VXq3bu31qxZo5CQH3/dcubMmXr44YeVk5OjDh066LrrrtPatWt16aWX1vt64uLi9OKLL2rdunXq1KmTVqxYoezsbK990tLStHLlSr3xxhvq2rWrBgwYcEG/jBoZGal33nlHhw4dUo8ePXTLLbcoPT1dCxcu9PFqfr6GNmNQUJD+9re/aevWrerYsaMmT56sv/zlL57bo6Oj9fe//13bt2/Xr3/9az300EO1fvCoy+c2EBw8eNBnr1f7y6nHvtO3U39NJzw8XO+9955GjRqlqVOnql27drruuusUHBysTZs2eX5HTpIGDRqkZcuW6dprr1WHDh00YcIEDRo0SK+88orf1+Cw6vrK4UUgLS1NXbt2bRB/yik7O1uvv/46T8PgoldaWqpLL71U27Zt4091+UlDeezzx+eaM7kz/PWvf1V0dHS9vdvxTLt371Z0dLRmz55ty8cHcHGy+7Fv8ODBtf66ii9wJneab7/9VseOHZMktWnTRmFhYfU+Q3V1tUpLSyVJTqfTljdtAA0JZ3L+1xAe+/w1A5EDABiLpysBAMYicgAAYxE5AICxiJwfuFwuZWdny+Vy2T2Kz7CmwMCaAoOJa5Ia5rp444kfVFZWKjY2VhUVFX7/bf76wpoCA2sKDCauSWqY6+JMDgBgLCIHADBWiN0D1Be32619+/apUaNGnj8a6i+VlZVe/2sC1hQYWFNgMHFNUv2ty7IsHT58WK1atVJQ0PnP1S6a1+T27t3LXw8BAIPs2bNHrVu3Pu8+F82ZXKNGjSRJaa3GKiSo/v9kjb+UDjcv3JHfmfdzV+N/HLN7BJ9zmPdpkiQ5TtbYPYLPlXeJtnsEn6o5cVzFL870PK6fz0UTuVNPUYYEhSkkyGnzNL4T7Ay3ewSfCw4z79EzJMS8NRkbOcu8yAWHmfc4IalOLz3xxhMAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGCsgIhcWlqaJk2aZPcYAIAAE2L3AHWxatUqhYaGSpKSk5M1adIkogcA+EkBEbm4uDi7RwAABKCAeroyLS1N33zzjSZPniyHwyGHw2H3aACABiwgInfKqlWr1Lp1a82YMUNlZWUqKys7574ul0uVlZVeGwDg4hJQkYuLi1NwcLAaNWqkhIQEJSQknHPfnJwcxcbGerbExMR6nBQA0BAEVOQuRFZWlioqKjzbnj177B4JAFDPAuKNJz+H0+mU0+m0ewwAgI0C7kwuLCxMNTU1do8BAAgAARe55ORkffTRR/r222918OBBu8cBADRgARe5GTNmqLS0VG3btlWzZs3sHgcA0IAFxGtyH3zwgee/e/XqpcLCQvuGAQAEjIA7kwMAoK6IHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGCsELsHqG81+7+XwxFq9xg+E7030e4RfO5ogsPuEXyueq9532oRew/bPYJfOKqO2j2Czx3qEmH3CD7lPlZT5305kwMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADBWwEXu1VdfVadOnRQREaH4+HhlZGToyJEjdo8FAGiAQuwe4EKUlZVp+PDhmjNnjm688UYdPnxYH3/8sSzLqrWvy+WSy+XyXK6srKzPUQEADUDARa66ulo33XSTkpKSJEmdOnU66745OTmaPn16fY4HAGhgAurpyi5duig9PV2dOnVSZmamnn76af3www9n3TcrK0sVFRWebc+ePfU8LQDAbgEVueDgYL377rt66623lJqaqgULFqh9+/b65z//WWtfp9OpmJgYrw0AcHEJqMhJksPhUJ8+fTR9+nRt27ZNYWFhWr16td1jAQAaoIB6TS4/P195eXm69tpr1bx5c+Xn5+vAgQPq0KGD3aMBABqggIpcTEyMPvroIz355JOqrKxUUlKS5s2bp8GDB9s9GgCgAQqoyHXo0EFvv/223WMAAAJEwL0mBwBAXRE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFghdg9Q3xzOMDkcYXaP4TON/3HU7hF8rsmocrtH8Lm9QW3sHsHnWp6IsnsEv3AePW73CD6XOnO33SP4VLX7hPbWcV/O5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYq8FH7uTJk3aPAAAIUD6N3NKlS9WqVSu53W6v63//+9/rT3/6kyRpzZo16tatm8LDw3XZZZdp+vTpqq6u9uzrcDi0aNEi/e53v1NUVJRmzZqldu3aae7cuV7HLCgokMPh0M6dO325BACAQXwauczMTJWXl+v999/3XHfo0CG9/fbbGjFihD7++GONGjVKEydOVHFxsZYsWaLnn39ejz76qNdxsrOzdeONN2r79u0aO3as/vSnP+m5557z2ue5557TNddco3bt2p11FpfLpcrKSq8NAHBx8WnkmjRposGDB2v58uWe61599VU1bdpU/fv31/Tp0/XnP/9Zo0eP1mWXXaaBAwdq5syZWrJkiddxbr31Vt1+++267LLL1KZNG40ZM0Y7duzQ5s2bJf34FOby5cs9Z4dnk5OTo9jYWM+WmJjoy6UCAAKAz1+TGzFihF577TW5XC5J0ksvvaRhw4YpKChIhYWFmjFjhqKjoz3buHHjVFZWpqNHj3qO0b17d69jtmrVSr/97W/13//935Kkv//973K5XMrMzDznHFlZWaqoqPBse/bs8fVSAQANXIivDzh06FBZlqW1a9eqR48e+vjjjzV//nxJUlVVlaZPn66bbrqp1v3Cw8M9/x0VFVXr9jvuuEMjR47U/Pnz9dxzz+mPf/yjIiMjzzmH0+mU0+n0wYoAAIHK55ELDw/XTTfdpJdeekk7d+5U+/bt1a1bN0lSt27dtGPHjnO+jnY+119/vaKiorRo0SK9/fbb+uijj3w9OgDAMD6PnPTjU5ZDhgxRUVGRbrvtNs/1jzzyiIYMGaI2bdrolltu8TyF+eWXX2rWrFnnPWZwcLDGjBmjrKwsXX755erdu7c/RgcAGMQvvyc3YMAAxcXFaceOHbr11ls91w8aNEhvvvmm1q9frx49eqhXr16aP3++kpKS6nTcsWPH6sSJE7r99tv9MTYAwDB+OZMLCgrSvn37znrboEGDNGjQoHPe17Ksc9727bffKjQ0VKNGjfrFMwIAzOeXyPmay+XSgQMHlJ2drczMTLVo0cLukQAAAaDB/1kvSVqxYoWSkpL0r3/9S3PmzLF7HABAgAiIyI0ZM0Y1NTXaunWrLrnkErvHAQAEiICIHAAAPweRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGCsELsHqHc1NZKjxu4pfCZoS4ndI/jcweVX2j2Cz/Ua94XdI/jcB8072j2CX7SffdTuEXyu5sABu0fwqWrrZJ335UwOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxmqQkRszZoxuuOGG8+6TlpamSZMm1cs8AIDA1CAjBwCALxA5AICx/BY5t9utOXPmqF27dnI6nWrTpo0effRRSdL27ds1YMAARUREKD4+XuPHj1dVVdU5j3XkyBGNGjVK0dHRatmypebNm+evsQEABvFb5LKysvTYY4/p4YcfVnFxsZYvX64WLVroyJEjGjRokJo0aaItW7Zo5cqVys3N1T333HPOYz3wwAP68MMPtWbNGq1fv14ffPCBPv/88/N+fJfLpcrKSq8NAHBxCfHHQQ8fPqynnnpKCxcu1OjRoyVJbdu21W9+8xs9/fTTOn78uF544QVFRUVJkhYuXKihQ4fq8ccfV4sWLbyOVVVVpWeffVYvvvii0tPTJUnLli1T69atzztDTk6Opk+f7ofVAQAChV/O5EpKSuRyuTxROvO2Ll26eAInSX369JHb7daOHTtq7b9r1y6dOHFCPXv29FwXFxen9u3bn3eGrKwsVVRUeLY9e/b8ghUBAAKRX87kIiIi/HHYC+J0OuV0Ou0eAwBgI7+cyV1++eWKiIhQXl5erds6dOigwsJCHTlyxHPdJ598oqCgoLOenbVt21ahoaHKz8/3XPfDDz/oq6++8sfoAACD+OVMLjw8XFOmTNGDDz6osLAw9enTRwcOHFBRUZFGjBihadOmafTo0crOztaBAwc0YcIEjRw5stbrcZIUHR2tsWPH6oEHHlB8fLyaN2+uhx56SEFB/PYDAOD8/BI5SXr44YcVEhKiRx55RPv27VPLli115513KjIyUu+8844mTpyoHj16KDIyUjfffLOeeOKJcx7rL3/5i6qqqjR06FA1atRI9913nyoqKvw1OgDAEA7Lsiy7h6gPlZWVio2N1YDwPyjEEWb3OD5j1bjtHsHnDo660u4RfK7LuO12j+BzH3za0e4R/KL97F12j+BzNQcO2D2CT1VbJ/WB1qiiokIxMTHn3Zfn/AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLFC7B6gvjnCwuRwhNk9hu+cOGH3BD7X9Isqu0fwuZInO9o9gs/temKx3SP4xcDXxtg9gs8FHThg9wi24UwOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxvJp5NLS0jRp0iRfHhIAgJ+NMzkAgLEadOROnDhh9wgAgADm88i53W49+OCDiouLU0JCgrKzsz23/etf/9Idd9yhZs2aKSYmRgMGDFBhYaHn9uzsbHXt2lXPPPOMLr30UoWHh9fpfgAAnI3PI7ds2TJFRUUpPz9fc+bM0YwZM/Tuu+9KkjIzM/X999/rrbfe0tatW9WtWzelp6fr0KFDnvvv3LlTr732mlatWqWCgoI63+9MLpdLlZWVXhsA4OIS4usDdu7cWdOmTZMkXX755Vq4cKHy8vIUERGhzZs36/vvv5fT6ZQkzZ07V6+//rpeffVVjR8/XtKPT1G+8MILatasmSRpw4YNdbrfmXJycjR9+nRfLw8AEED8ErnTtWzZUt9//70KCwtVVVWl+Ph4r9uPHTumXbt2eS4nJSV5Aiepzvc7U1ZWlu69917P5crKSiUmJv6sNQEAApPPIxcaGup12eFwyO12q6qqSi1bttQHH3xQ6z6NGzf2/HdUVJTXbXW935mcTqfnzA8AcHHyeeTOpVu3btq/f79CQkKUnJzs9/sBAFBvv0KQkZGh3r1764YbbtD69etVWlqqjRs36qGHHtJnn33m8/sBAFBvkXM4HFq3bp2uueYa3X777UpJSdGwYcP0zTffqEWLFj6/HwAADsuyLLuHqA+VlZWKjY1VesxtCnGE2T2Oz1gG/sK81elyu0fwucq2UT+9U4DZ+MRiu0fwi4F/GGP3CD4XtKHA7hF8qto6qQ+0RhUVFYqJiTnvvg36L54AAPBLEDkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGAsIgcAMBaRAwAYi8gBAIxF5AAAxiJyAABjETkAgLGIHADAWEQOAGCsELsHqG8nO14qKyTc7jF8JvjT7XaP4HMnYsPsHsHnnD9U2z2Cz7V9+U67R/AL9+gau0fwuZRPHHaP4GMOyarbnpzJAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGCvE7gH8xeVyyeVyeS5XVlbaOA0AwA7Gnsnl5OQoNjbWsyUmJto9EgCgnhkbuaysLFVUVHi2PXv22D0SAKCeGft0pdPplNPptHsMAICNjD2TAwCAyAEAjBWwkVu4cKHS09PtHgMA0IAFbOQOHjyoXbt22T0GAKABC9jIZWdnq7S01O4xAAANWMBGDgCAn0LkAADGInIAAGMROQCAsYgcAMBYRA4AYCwiBwAwFpEDABiLyAEAjEXkAADGInIAAGMROQCAsYgcAMBYRA4AYCwiBwAwFpEDABiLyAEAjEXkAADGInIAAGOF2D1AfTucFK7gsHC7x/CZpjub2j2Cz4VVnLB7BJ8Lqjpu9wg+1/Rz8772JKnZ2L12j+Bz1WFhdo/gUw7LIbnqti9ncgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADDWBUUuLS1NDodDDodDBQUFfhqp4c8AAAgMF3wmN27cOJWVlaljx44qLS31BOfMbdOmTZ77HDt2TNOmTVNKSoqcTqeaNm2qzMxMFRUVeR376NGjysrKUtu2bRUeHq5mzZqpX79+WrNmjWefVatWafPmzb9gyQCAi0XIhd4hMjJSCQkJXtfl5ubqiiuu8LouPj5ekuRyuZSRkaHdu3dr3rx56tmzp7777jvl5OSoZ8+eys3NVa9evSRJd955p/Lz87VgwQKlpqaqvLxcGzduVHl5uee4cXFxqqysvOCFAgAuPhccubOJj4+vFb5TnnzySX366afatm2bunTpIklKSkrSa6+9pp49e2rs2LH68ssv5XA49MYbb+ipp57S9ddfL0lKTk7WlVde6YsRAQAXIb+/8WT58uUaOHCgJ3CeDxwUpMmTJ6u4uFiFhYWSpISEBK1bt06HDx/+xR/X5XKpsrLSawMAXFx8Ermrr75a0dHRXtspX331lTp06HDW+526/quvvpIkLV26VBs3blR8fLx69OihyZMn65NPPvlZM+Xk5Cg2NtazJSYm/qzjAAACl08i9/LLL6ugoMBrO51lWXU6zjXXXKOvv/5aeXl5uuWWW1RUVKS+fftq5syZFzxTVlaWKioqPNuePXsu+BgAgMDmk9fkEhMT1a5du7PelpKSopKSkrPedur6lJQUz3WhoaHq27ev+vbtqylTpmjWrFmaMWOGpkyZorCwsDrP5HQ65XQ6L2AVAADT+P01uWHDhik3N9fzutspbrdb8+fPV2pqaq3X606Xmpqq6upqHT9+3N+jAgAM45MzufLycu3fv9/rusaNGys8PFyTJ0/WmjVrNHToUK9fIZg9e7ZKSkqUm5srh8Mh6cdf9B4+fLi6d++u+Ph4FRcXa+rUqerfv79iYmJ8MSoA4CLik8hlZGTUum7FihUaNmyYwsPD9d5772n27NmaOnWqvvnmGzVq1Ej9+/fXpk2b1LFjR899Bg0apGXLlmnq1Kk6evSoWrVqpSFDhuiRRx7xxZgAgIvML4pccnJynd5UEhkZqVmzZmnWrFnn3S8rK0tZWVm/ZCQAADwu+DW5v/71r4qOjtb27dv9Mc9PGjx4cK2/rgIAwNlc0JncSy+9pGPHjkmS2rRp45eBfsozzzxj+wwAgMBwQZG75JJL/DVHQM0AAAgM/P/JAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADAWkQMAGIvIAQCMReQAAMYicgAAYxE5AICxiBwAwFhEDgBgLCIHADDWBf3/yQUyy7IkSTUnj9s8iW9Vu0/YPYLP1dSY9TmSpKAal90j+FzNCfM+T5J08oiB31PWSbtH8Knq/7+eU4/r5+Ow6rKXAfbu3avExES7xwAA+MiePXvUunXr8+5z0UTO7XZr3759atSokRwOh18/VmVlpRITE7Vnzx7FxMT49WPVF9YUGFhTYDBxTVL9rcuyLB0+fFitWrVSUND5X3W7aJ6uDAoK+sni+1pMTIxRX8ASawoUrCkwmLgmqX7WFRsbW6f9eOMJAMBYRA4AYCwi5wdOp1PTpk2T0+m0exSfYU2BgTUFBhPXJDXMdV00bzwBAFx8OJMDABiLyAEAjEXkAADGInIAAGMROQCAsYgcAMBYRA4AYCwiBwAw1v8D/mHGq6BrU3EAAAAASUVORK5CYII="
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 560x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAGkCAYAAAAsb2x+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAi4ElEQVR4nO3de1iUdf7/8ddwmgGBIZAEk2QTKNE8FK1peWjF2kyr3WLT7YSVbW2bh7JvYaVWJv28dNVfXR1sN7NWrW3TtYO2haWbeUoNa9WyMJQKMzHB4yjM5/fH/pyvhMeUuWc+PB/XdV85wz037w/CPL1nhslljDECAADWiHB6AAAAcGoRdwAALEPcAQCwDHEHAMAyxB0AAMsQdwAALEPcAQCwDHEHAMAyxB0AAMsQ96Po1auXXC6XXC6XSktLHZmhvLw8MEOnTp0cmQE4GQsXLpTL5dKOHTucHuWkjRkz5pg/h4WFhbr66quDMg9OTCjcpwdrBuJ+DIMHD1ZlZaXat29fL7Qul0sxMTHKysrS2LFj9dN38V27dq1+97vfKTU1VW63Wzk5ORo1apT27NlTb781a9boyiuv1Omnny6Px6PMzExdd9112rp1qyQpIyNDlZWVuvfee4O25lPt4NfNqR8mOKtbt26qrKyU1+t1epSTNmLECC1YsMDpMXASjnaffui2bNmywG327t2r0aNHKycnR263W82bN1dBQYHWrl1b79h79uxRUVGR2rRpI4/Ho9TUVPXs2VNz584N7DN79mytWLGi0dcZ1eifIczFxcUpLS2t3nUlJSVq166dfD6fFi9erNtuu03p6em69dZbJUnLli1Tfn6+8vPz9fbbb6tFixZasWKF7r33Xi1YsEAffPCBYmJi9MMPP6h3797q16+f/vWvfykpKUnl5eV64403tHv3bklSZGSk0tLSFB8fH/S1A6dCTExMg5+hcBUfH8/PYpg72n36oVJSUiRJPp9P+fn52rx5syZOnKguXbro+++/V3Fxsbp06aKSkhJdeOGFkqQ77rhDy5cv15NPPqnc3FxVVVVpyZIlqqqqChw3OTlZNTU1jbxKSQZH1LNnTzN06NDA5a+//tpIMp988km9/Xr37m3++Mc/GmOM8fv9Jjc31+Tl5Zm6urp6+5WWlhqXy2WeeOIJY4wxc+bMMVFRUebAgQPHnGX06NGmY8eOJ7Wek1VXV2fGjRtnMjMzjcfjMR06dDCvvfaaMcaY7du3m9///vemefPmxuPxmKysLPPCCy8YY4yRVG/r2bOnMcaYFStWmPz8fJOSkmISExNNjx49zKpVq5xa3s/yc78m4axnz57mT3/6kxk6dKhJSkoyp59+upk6darZtWuXKSwsNPHx8aZNmzZm3rx5xhhjPvjgAyPJ/Pjjj84Ofhyee+45k56e3uBn98orrzSDBg1q8HNYW1trhg8fbrxer0lOTjb33Xefuemmm8xVV10V2Gf+/PnmoosuCuxzxRVXmK+++ipIK8Khjvc+/VBPPPGEcblcprS0tN71dXV1Ji8vz+Tm5hq/32+MMcbr9ZoXX3zxmHMcz+c9WTwsf5JWrlypVatWqUuXLpKk0tJSrVu3Tvfcc48iIup/eTt27Kj8/HzNmjVLkpSWlqba2lrNmTOnwcP6oai4uFgvvfSSnn32Wa1du1bDhw/XDTfcoEWLFunhhx/WunXrNH/+fK1fv17PPPOMmjdvLkmBh6BKSkpUWVmp2bNnS5J27typm2++WYsXL9ayZcuUnZ2tvn37aufOnY6t8UT93K9JuJs+fbqaN2+uFStW6O6779add96pgoICdevWTatXr9all16qG2+8scHTUKGuoKBAVVVV+uCDDwLXbd++Xe+8846uv/76BvtPnDhRL774ol544QUtXrxY27dv15w5c+rts3v3bt1zzz1auXKlFixYoIiICP3mN7+R3+9v9PXg5M2cOVN9+vRRx44d610fERGh4cOHa926dVqzZo2k/96nz5s3LzTuwxrtnw0WONK/8mJjY02zZs1MdHS0kWRuv/32wD6vvPLKUf9FNmTIEBMbGxu4PHLkSBMVFWWSk5PNr3/9azN+/HizZcuWBrdz+sx93759Ji4uzixZsqTe9bfeeqsZOHCg6d+/vxk0aNBhb3u8/0qtq6szCQkJ5s033zxVYzeqk/mahLOePXuaiy++OHC5trbWNGvWzNx4442B6yorK40ks3Tp0rA6czfGmKuuusrccsstgcvPPfecadmypamrq2vwc5ienm7Gjx8fuHzgwAHTqlWremfuP/XDDz8YSeazzz5rjPFxFMe6Tz90O8jj8dS7zaFWr15tJJlXX33VGGPMokWLTKtWrUx0dLTJy8szw4YNM4sXL25wO87cQ9Srr76q0tJSrVmzRn//+981d+5cPfDAA/X2Mcd5Jv74449ry5YtevbZZ9WuXTs9++yzOuecc/TZZ581xug/21dffaU9e/aoT58+gecd4+Pj9dJLL6msrEx33nmnXnnlFXXq1En/8z//oyVLlhzzmN9//70GDx6s7Oxseb1eJSYmateuXdq8eXMQVnTyGuNrEi46dOgQ+HNkZKRSUlJ07rnnBq5r0aKFJAVeGBpOrr/+er3++uvy+XySpBkzZmjAgAENHomrrq5WZWVl4FE7SYqKilJeXl69/b788ksNHDhQZ511lhITE5WZmSlJYfN93hQcvE8/dDvU8d6f9+jRQxs3btSCBQt07bXXau3aterevbsee+yxRpj66HhB3c+QkZGhrKwsSVLbtm1VVlamhx9+WGPGjFFOTo4kaf369ercuXOD265fvz6wz0EpKSkqKChQQUGBxo0bp86dO2vChAmaPn164y/mOO3atUuS9Pbbb+uMM86o9zG3262MjAxt2rRJ8+bN03vvvafevXvrrrvu0oQJE454zJtvvllVVVWaMmWKWrduLbfbra5du2r//v2NupZTpTG+JuEiOjq63mWXy1XvOpfLJUlh+dBz//79ZYzR22+/rQsuuEAffvihJk2adFLHa926tZ5//nm1bNlSfr9f7du3D5vv86bg0Pv0n8rJydH69esP+7GD1x96nx4dHa3u3bure/fuuv/++zV27Fg9+uijuv/++xUTE3Pqhz8CztxPgcjISNXW1mr//v3q1KmTzjnnHE2aNKnBHduaNWtUUlKigQMHHvFYMTExatOmTeDV8qEiNzdXbrdbmzdvVlZWVr0tIyNDkpSamqqbb75Zf/vb3zR58mRNnTpVkgLf0HV1dfWO+dFHH2nIkCHq27ev2rVrJ7fbrW3btgV3YSfhZL4mCF0ej0e//e1vNWPGDM2aNUtnn322zjvvvAb7eb1epaena/ny5YHramtrtWrVqsDlqqoqffHFF3rooYfUu3dvtW3bVj/++GNQ1oFTY8CAASopKQk8r36Q3+/XpEmTlJub2+D5+EPl5uaqtrZW+/bta+xR6+HM/WeoqqrSli1bVFtbq88++0xTpkzRJZdcosTEREnSX//6V/Xp00fXXHONioqKlJaWpuXLl+vee+9V165dNWzYMEnSW2+9pVdeeUUDBgxQTk6OjDF68803NW/ePE2bNs3BFTaUkJCgESNGaPjw4fL7/br44otVXV2tjz76SImJiSorK9P5558f+BXBt956S23btpUknX766YqNjdU777yjVq1ayePxyOv1Kjs7Wy+//LLy8vJUU1Oj++67T7GxsQ6v9PidzNcEoe36669Xv379tHbtWt1www1H3G/o0KF64oknlJ2drXPOOUd//vOf671Zz2mnnaaUlBRNnTpV6enp2rx5c4On8MLVU089pTlz5ljxe/8H79MPlZSUJI/Ho+HDh2vu3Lnq379/vV+FGzdunNavX6+SkpLAI1W9evXSwIEDlZeXp5SUFK1bt04jR46s14egabRn8y1wpBdfHNwiIyNNq1atzODBg83WrVvr3fbTTz8111xzjUlOTjbR0dGmTZs25qGHHjK7d+8O7FNWVmYGDx5scnJyTGxsrElKSjIXXHCBmTZtWoNZnH5BnTH//TW/yZMnm7PPPttER0eb1NRUc9lll5lFixaZxx57zLRt29bExsaa5ORkc9VVV5mNGzcGbvv888+bjIwMExEREfhVuNWrV5u8vDzj8XhMdna2ee2110zr1q3NpEmTnFngz3AyX5Nw9dOfC2PMYf/eJJk5c+aE3QvqjPnvizvT09ONJFNWVha4/qc/hwcOHDBDhw41iYmJJikpydxzzz0NfhXuvffeM23btjVut9t06NDBLFy4MPC1CWejR482rVu3dnqME3Ks+/RDt1mzZgX22717t3nwwQdNVlaWiY6ONsnJyeaaa65p8KLIcePGma5du5rk5GTj8XjMWWedZYYMGWK2bdtWb79gvKDOZUwY/A6WQ3r16qVOnTpp8uTJTo+iMWPG6J///Cfv8gYAP1Oo3KeXl5frF7/4hT755JNGe1txnnM/hqefflrx8fGOvXp98+bNio+P17hx4xz5/ABgE6fv0y+//PIG74bXGDhzP4pvv/1We/fulSSdeeaZQX2l40G1tbUqLy+X9L+vwAYAnLhQuE8P1gzEHQAAy/CwPAAAliHuAABYhrgDAGAZ4h4EPp9PY8aMCbxXdVPBull3U8C6WXco4gV1QVBTUyOv16vq6urgv0uRg1g3624KWDfrDkWcuQMAYBniDgCAZZrM/zjG7/fru+++U0JCQuBN/oOlpqam3n+bCtbNupsC1s26g8UYo507d6ply5aKiDj6uXmTec79m2++4d3dAABhr6KiQq1atTrqPk3mzD0hIUGS1MP9G0W5oh2eJrg2/J/Gfx/jUBT/VdP6ez6o1WvlTo/gCP/u3U6P4IwDB5yewBGulmlOjxB0tX6fFpU/F+jZ0TSZuB98KD7KFa0oV/DfT9hJEbEep0dwRKS7acY9KqJpfX8f5Hc1zcgpyE8zhgpXpNvpERxzPE8t84I6AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDKOxX3hwoVyuVzasWPHEfcZM2aMOnXqFLSZAACwQdDi3qtXLw0bNuyEbjNixAgtWLCgcQYCAMBSUU4PcDTx8fGKj493egwAAMJKUM7cCwsLtWjRIk2ZMkUul0sul0vl5eWSpFWrVikvL09xcXHq1q2bvvjii8Dtfvqw/MKFC/XLX/5SzZo1U1JSki666CJt2rQpGEsAACBsBCXuU6ZMUdeuXTV48GBVVlaqsrJSGRkZkqQHH3xQEydO1MqVKxUVFaVbbrnlsMeora3V1VdfrZ49e+rTTz/V0qVLdfvtt8vlch12f5/Pp5qamnobAABNQVAelvd6vYqJiVFcXJzS0tIkSZ9//rkk6fHHH1fPnj0lSQ888ICuuOIK7du3Tx6Pp94xampqVF1drX79+qlNmzaSpLZt2x7xcxYXF+uRRx5pjOUAABDSHP9VuA4dOgT+nJ6eLknaunVrg/2Sk5NVWFioyy67TP3799eUKVNUWVl5xOMWFRWpuro6sFVUVJz64QEACEGOxz06Ojrw54MPsfv9/sPuO23aNC1dulTdunXTq6++qpycHC1btuyw+7rdbiUmJtbbAABoCoIW95iYGNXV1Z30cTp37qyioiItWbJE7du318yZM0/BdAAA2CNocc/MzNTy5ctVXl6ubdu2HfHs/Ei+/vprFRUVaenSpdq0aZPeffddffnll0d93h0AgKYoaHEfMWKEIiMjlZubq9TUVG3evPmEbh8XF6fPP/9c11xzjXJycnT77bfrrrvu0h/+8IdGmhgAgPDkMsYYp4cIhpqaGnm9Xv3K8ztFuWKcHieoPp9yrtMjOCJhQ/Sxd7JQxsyNTo/gCP+u3U6P4IwDB5yewBGuVulOjxB0tXU+Ldj4f1VdXX3M15E5/oI6AABwahF3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDJRTg8QbP79tfK7XE6PEVRnP7/X6REc8cUdxukRHGGSvU6P4IiIqCZ3dyZJ8v+wzekRHLH7nFSnRwi62gP7pI3Hty9n7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWCbk496rVy8NGzbM6TEAAAgbUU4PcCyzZ89WdHS002MAABA2Qj7uycnJTo8AAEBYCauH5Z9++mllZ2fL4/GoRYsWuvbaa50dDgCAEBTyZ+4HrVy5UkOGDNHLL7+sbt26afv27frwww+PuL/P55PP5wtcrqmpCcaYAAA4LmzivnnzZjVr1kz9+vVTQkKCWrdurc6dOx9x/+LiYj3yyCNBnBAAgNAQ8g/LH9SnTx+1bt1aZ511lm688UbNmDFDe/bsOeL+RUVFqq6uDmwVFRVBnBYAAOeETdwTEhK0evVqzZo1S+np6Ro1apQ6duyoHTt2HHZ/t9utxMTEehsAAE1B2MRdkqKiopSfn6/x48fr008/VXl5ud5//32nxwIAIKSEzXPub731ljZu3KgePXrotNNO07x58+T3+3X22Wc7PRoAACElbOKelJSk2bNna8yYMdq3b5+ys7M1a9YstWvXzunRAAAIKSEf94ULFx72zwAA4PDC6jl3AABwbMQdAADLEHcAACxD3AEAsAxxBwDAMsQdAADLEHcAACxD3AEAsAxxBwDAMsQdAADLEHcAACxD3AEAsAxxBwDAMsQdAADLEHcAACxD3AEAsAxxBwDAMsQdAADLEHcAACxD3AEAsAxxBwDAMsQdAADLEHcAACxD3AEAsEyU0wOg8bnWbXR6BEe0/XMrp0dwRPJftjo9giO23pfp9AiOiNq/3+kRHNHsoy+dHiHoas3x/11z5g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZR+Leq1cv3X333Ro2bJhOO+00tWjRQs8//7x2796tQYMGKSEhQVlZWZo/f76MMcrKytKECRPqHaO0tFQul0tfffWVE0sAACBkOXbmPn36dDVv3lwrVqzQ3XffrTvvvFMFBQXq1q2bVq9erUsvvVQ33nij9u7dq1tuuUXTpk2rd/tp06apR48eysrKOuzxfT6fampq6m0AADQFjsW9Y8eOeuihh5Sdna2ioiJ5PB41b95cgwcPVnZ2tkaNGqWqqip9+umnKiws1BdffKEVK1ZIkg4cOKCZM2fqlltuOeLxi4uL5fV6A1tGRkawlgYAgKMci3uHDh0Cf46MjFRKSorOPffcwHUtWrSQJG3dulUtW7bUFVdcoRdeeEGS9Oabb8rn86mgoOCIxy8qKlJ1dXVgq6ioaKSVAAAQWhyLe3R0dL3LLper3nUul0uS5Pf7JUm33XabXnnlFe3du1fTpk3Tddddp7i4uCMe3+12KzExsd4GAEBTEOX0AMerb9++atasmZ555hm98847+ve//+30SAAAhKSw+VW4yMhIFRYWqqioSNnZ2eratavTIwEAEJLCJu6SdOutt2r//v0aNGiQ06MAABCyHHlYfuHChQ2uKy8vb3CdMabe5W+//VbR0dG66aabGmkyAADCX1g85+7z+fTDDz9ozJgxKigoCLySHgAANBQWD8vPmjVLrVu31o4dOzR+/HinxwEAIKSFRdwLCwtVV1enVatW6YwzznB6HAAAQlpYxB0AABw/4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGCZKKcHCLbI07yKjIhxeoygqtu+w+kRnLH+S6cncMT2wrOcHsERG//gcXoER/hjfuH0CI7Ivnu50yMEXZ05cNz7cuYOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBliDsAAJYh7gAAWIa4AwBgGeIOAIBlHIl7r169NGzYMElSZmamJk+eHPjYli1b1KdPHzVr1kxJSUlOjAcAQFiLcnqAjz/+WM2aNQtcnjRpkiorK1VaWiqv1+vgZAAAhCfH456amlrvcllZmc4//3xlZ2c7NBEAAOHN8efcD31YPjMzU6+//rpeeukluVwuFRYWSpJ27Nih2267TampqUpMTNSvfvUrrVmzxrmhAQAIYY6fuR/q448/1k033aTExERNmTJFsbGxkqSCggLFxsZq/vz58nq9eu6559S7d29t2LBBycnJhz2Wz+eTz+cLXK6pqQnKGgAAcJrjZ+6HSk1NldvtVmxsrNLS0uT1erV48WKtWLFCr732mvLy8pSdna0JEyYoKSlJ//jHP454rOLiYnm93sCWkZERxJUAAOCckIr74axZs0a7du1SSkqK4uPjA9vXX3+tsrKyI96uqKhI1dXVga2ioiKIUwMA4JyQelj+cHbt2qX09HQtXLiwwceO9qtybrdbbre78QYDACBEhXzczzvvPG3ZskVRUVHKzMx0ehwAAEJeyD8sn5+fr65du+rqq6/Wu+++q/Lyci1ZskQPPvigVq5c6fR4AACEnJCPu8vl0rx589SjRw8NGjRIOTk5GjBggDZt2qQWLVo4PR4AACHHZYwxTg8RDDU1NfJ6veqdMkhRETFOjxNUddt3OD2CM4zf6QkcEZl9ltMjOGLDH1KPvZOF/DFN4i68gey7lzs9QtDVmgNaqLmqrq5WYmLiUfcN+TN3AABwYog7AACWIe4AAFiGuAMAYBniDgCAZYg7AACWIe4AAFiGuAMAYBniDgCAZYg7AACWIe4AAFiGuAMAYBniDgCAZYg7AACWIe4AAFiGuAMAYBniDgCAZYg7AACWIe4AAFiGuAMAYBniDgCAZYg7AACWIe4AAFiGuAMAYJkopwcIuuanSZFup6cIqohdu50ewRH+ffucHsERdRvKnB7BEdkPVzo9giPmf7XE6REc8esRXZweIehcJkLyHd++nLkDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGCZKKcHaCw+n08+ny9wuaamxsFpAAAIHmvP3IuLi+X1egNbRkaG0yMBABAU1sa9qKhI1dXVga2iosLpkQAACAprH5Z3u91yu91OjwEAQNBZe+YOAEBTFdZxf+qpp9S7d2+nxwAAIKSEddy3bdumsrIyp8cAACCkhHXcx4wZo/LycqfHAAAgpIR13AEAQEPEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALBMlNMDBNuX9zZTRJzH6TGC6pxRLZwewRGm4lunR3CEy+12egRHRKSmOD2CIzo//kenR3BEmmu10yMEnesE9uXMHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAy5xQ3Hv16iWXyyWXy6XS0tJGGin0ZwAAIJSd8Jn74MGDVVlZqfbt26u8vDwQ2p9uy5YtC9xm7969Gj16tHJycuR2u9W8eXMVFBRo7dq19Y69Z88eFRUVqU2bNvJ4PEpNTVXPnj01d+7cwD6zZ8/WihUrTmLJAADYLepEbxAXF6e0tLR615WUlKhdu3b1rktJSZEk+Xw+5efna/PmzZo4caK6dOmi77//XsXFxerSpYtKSkp04YUXSpLuuOMOLV++XE8++aRyc3NVVVWlJUuWqKqqKnDc5ORk1dTUnPBCAQBoKk447oeTkpLSIPgHTZ48WUuXLtUnn3yijh07SpJat26t119/XV26dNGtt96q//znP3K5XHrjjTc0ZcoU9e3bV5KUmZmp888//1SMCABAk9HoL6ibOXOm+vTpEwh74BNHRGj48OFat26d1qxZI0lKS0vTvHnztHPnzpP+vD6fTzU1NfU2AACaglMS927duik+Pr7edtCGDRvUtm3bw97u4PUbNmyQJE2dOlVLlixRSkqKLrjgAg0fPlwfffTRz5qpuLhYXq83sGVkZPys4wAAEG5OSdxfffVVlZaW1tsOZYw5ruP06NFDGzdu1IIFC3Tttddq7dq16t69ux577LETnqmoqEjV1dWBraKi4oSPAQBAODolz7lnZGQoKyvrsB/LycnR+vXrD/uxg9fn5OQErouOjlb37t3VvXt33X///Ro7dqweffRR3X///YqJiTnumdxut9xu9wmsAgAAOzT6c+4DBgxQSUlJ4Hn1g/x+vyZNmqTc3NwGz8cfKjc3V7W1tdq3b19jjwoAgBVOyZl7VVWVtmzZUu+6pKQkeTweDR8+XHPnzlX//v3r/SrcuHHjtH79epWUlMjlckn67xvUDBw4UHl5eUpJSdG6des0cuRIXXLJJUpMTDwVowIAYL1TEvf8/PwG182aNUsDBgyQx+PR+++/r3HjxmnkyJHatGmTEhISdMkll2jZsmVq37594DaXXXaZpk+frpEjR2rPnj1q2bKl+vXrp1GjRp2KMQEAaBJOKu6ZmZnH9WK5uLg4jR07VmPHjj3qfkVFRSoqKjqZkQAAaPJO+Dn3p59+WvHx8frss88aY55juvzyyxu8Gx4AAPhfJ3TmPmPGDO3du1eSdOaZZzbKQMfyl7/8xfEZAAAIZScU9zPOOKOx5girGQAACGX8/9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsQ9wBALAMcQcAwDLEHQAAyxB3AAAsE+X0AMFijJEk+ff6HJ4k+Gr9TW/NklRnDjg9giNcxuX0CI6IaKrf5/v3OT2CI2rNfqdHCLra/3+fdrBnR+Myx7OXBb755htlZGQ4PQYAACeloqJCrVq1Ouo+TSbufr9f3333nRISEuRyBffMpqamRhkZGaqoqFBiYmJQP7eTWDfrbgpYN+sOFmOMdu7cqZYtWyoi4ujPqjeZh+UjIiKO+S+dxpaYmNikfggOYt1NC+tuWlh3cHm93uPajxfUAQBgGeIOAIBliHsQuN1ujR49Wm632+lRgop1s+6mgHWz7lDUZF5QBwBAU8GZOwAAliHuAABYhrgDAGAZ4g4AgGWIOwAAliHuAABYhrgDAGAZ4g4AgGX+H/8u1rmdJ+/7AAAAAElFTkSuQmCC"
     },
     "metadata": {}
    },
    {
     "execution_count": 34,
     "output_type": "execute_result",
     "data": {
      "text/plain": "'this is my life .'"
     },
     "metadata": {}
    }
   ]
  }
 ]
}
